Aczid's blog

Verified user aczid with mean accuracy of: 0.585
Logged in successfully!

class CsvImporter

  attr_accessor :table_class

  require 'fastercsv'

  def initialize(filename)
    # CSV filename
    @filename = filename
    # Table column names array
    @table_columns = ['primary_key']
    puts "Parsing CSV file #{filename}"
    parse_file(@filename)
  end 

  # Returns sanitized name from filename.
  # Replaces dashes with underscores, removes slashes, removes .csv extension and prepends 'csvimport_'
  def self.table_name(filename)
    basename = File.basename(filename.to_s).to_s
    table_name = "csvimport_#{basename.gsub(/\.csv/, '').gsub(/-/, '_').gsub(/\//, '')}"
    table_name
  end 

  # Import CSV data into the database table using the ORM class
  def parse_file(filename)
    @parsed_file = FasterCSV.read(filename)
    analyze_header(@parsed_file.shift)
    create_table(CsvImporter.table_name(filename), @table_columns)
    n = 0 
    @parsed_file.each do | row |
      hash = row2hash(row)
      unless @table_class.first(:primary_key => hash[:primary_key])
        instance = @table_class.new(hash)
        if instance.save
          n+=1
          GC.start if n%50 == 0
        end
      end
    end
  end

  # Converts a row of CSV data to a ruby Hash.
  def row2hash(row)
    hash = {}
    row.size.times do |i|
      unless row[i].nil?
        hash[ @table_columns[i].to_sym ] = row[i]
      end
    end
    hash
  end

  # Analyzes CSV header and adds fields to @table_columns array
  def analyze_header(header)
    header.each do | column |
      # strips digit prefixes from CSV header and adds the result to 
      # table columns
      if column
        #column = "token_#{column.to_s}" unless column.to_s[0].is_a?(Integer)
        @table_columns.push column.to_s.gsub(/^\d+: /,'')
      end
    end
  end

  # Automagically creates an ORM class for the import using @table_columns array
  def create_table(name, columns)
    # creates a new table class with a primary_key property
    @table_class = Class.new do
      include DataMapper::Resource
      property :id, DataMapper::Types::Serial
      property :updated_at, DateTime
      property :primary_key, String
    end

    # set table name
    @table_class.storage_names[:default] = name
    # shift first element off because it is the primary key
    pk = columns.shift
    columns.each do | column |
      # Here, I know all the columns except the primary key are of type Float. You can extend this to add magic for determining the type of data.
      @table_class.property column.to_sym, Float, :precision => 11
    end

    # unshift PK back in place
    columns.unshift(pk)
    # dont destroy tables we already have
    unless @table_class.storage_exists?
      @table_class.auto_migrate!
    end
  end
end

class CsvImporter
  def self.load_class(name)
    @table_class = Class.new do
      # Again, these types are known to always be there
      include DataMapper::Resource
      property :id, DataMapper::Types::Serial
      property :updated_at, DateTime
    end 
    @table_class.storage_names[:default] = name
    if @table_class.storage_exists?
      desc = repository(:default).adapter.query("desc #{name}")
      desc.each do |field|
        case field.type
        when /DateTime/i
          klass = DateTime
        when /Float/i
          klass = Float
        else
          klass = String
        end
        klass = DataMapper::Types::Serial if field.id == "id"
        if klass == Float
          @table_class.property field.id.to_sym, klass, :precision => 11
        else
          @table_class.property field.id.to_sym, klass
        end
      puts "Created field with id #{field.id.to_sym}, class: #{klass}"
      end 
    end
    @table_class
  end
end

    rows = @table_class.all
    rows.each do | row |
      key = row.primary_key.to_sym
      @matches[key] = []
      row.instance_variables.each do | column |
        unless ['@id', '@repository','@primary_key','@original_values', '@new_record','@collection', '@updated_at'].include? column
          x = row.instance_variable_get(column)
          y = column.gsub(/@/, '') 
          @matches[key] << {:x => x, :y => y}
        end
      end 
      @matches[key] = @matches[key].sort_by { |match| match[:y] }
    end
    @matches

    n=0
    @parsed_file.each do | row |
      hash = row2hash(row)
      unless @table_class.first(:primary_key => hash[:primary_key])
        instance = @table_class.new(hash)
        if instance.save
          n+=1
          GC.start if n%50 == 0
        end
      end
    end

    values = {}
    @parsed_file[0].enum_with_index.map do |primary_key, idx|
      if primary_key
        pk = primary_key.to_sym
        @parsed_file.collect do |row|
          if row[0]
            values[pk] = {} unless values[pk].is_a?(Hash)
            values[pk][row[0].to_sym] = row[idx]
          end
        end
      end 
    end
    n = 0 
    values.keys.each do |key|
      if values[key]
        unless @table_class.first(:primary_key => key.to_s)
          instance = @table_class.new(values[key])
          instance.primary_key = key
          if instance.save
            n+=1
            GC.start if n%50 == 0
          end
        end
      end
    end

    @matches[primary_key.to_sym] = @table_class.all(:order => [primary_key.to_sym.desc])

 require 'mkmf'
# Create wrapper module
`swig -c++ -ruby -minherit -Wall -o units_wrap.cpp units.i`

# Since the SWIG runtime support library for Ruby
# depends on the Ruby library, make sure it's in the list
# of libraries.
$libs = append_library($libs, Config::CONFIG['RUBY_INSTALL_NAME'])

# Also, we need the c++ libraries
$libs = append_library($libs, "stdc++")

# Create the makefile
create_makefile('units') 

bin_PROGRAMS = example
example_SOURCES = example.cpp
noinst_unit_testsdir = .
noinst_unit_tests_DATA = units.so
EXTRA_DIST = extconf.rb autogen.sh
units.so:
     ruby extconf.rb
     make

#!/bin/sh

CXX=/usr/bin/g++
SRC_DIR=src
CFLAGS="-fPIC -fno-strict-aliasing -g -O2"
BIN_DIR=bin
RUBY_LIBS=/usr/lib/ruby/1.8/i486-linux
SWIG=/usr/bin/swig

cd $SRC_DIR

SRCS=`ls *.cpp | sed "s/.*_wrap\.cpp//"`
OBJS=`ls *.o`

`${CXX} -I. -I${RUBY_LIBS} ${CFLAGS} -c ${SRCS}`
`${SWIG} -c++ -ruby -Wall -o units_wrap.cpp units.i`
`${CXX} -I. -I${RUBY_LIBS} ${CFLAGS} -c units_wrap.cpp`
`${CXX} -shared -L. -rdynamic -Wl,-export-dynamic -L/usr/local/lib -o units.so ${OBJS} -lruby1.8 -lpthread -ldl -lcrypt -lm -lc`

require File.dirname(__FILE__) + '/spec_helper'
require 'units'

describe Units::FileReader do
  it "should be a constant on module Units" do
    Units.constants.should include('FileReader')
  end
end

describe Units::FileReader, ".new" do
  it "should create a new object of the type FileReader" do
    fr = Units::FileReader.new("file.txt")
    fr.filename.should_be "file.txt"
  end
end

describe Units::FileReader, "#openFile" do
  it "should open a file" do
    fr = Units::FileReader.new
    fr.openFile("file.txt").should_be true
    fr.filename.should_be "file.txt"
  end
end

describe Units::FileReader, "#closeFile" do
  it "should close a file" do
    fr = Units::FileReader.new
    fr.closeFile.should_be true
    fr.filename.should_be ""
  end
end

describe Units::FileReader, "#readChar" do
  it "should read a char" do
    fr = Units::FileReader.new("file.txt")
    char = fr.readChar
    char.should_be fr.ch
    char.is_a?(String).should_be true
    char.length.should_be 1
  end
end

describe Units::FileReader, "#readLine" do
  it "should read a line from the file" do
    fr = Units::FileReader.new("file.txt")
    line = fr.readLine
    line.is_a?(String).should_be true
    line.match(/^.*\n$/).should_not be_nil
  end
end

class ExampleWorker < BackgrounDRb::Worker::RailsBase
  def do_work(args)
    results[:some_string] = "I'm robust!"
  end
end
ExampleWorker.register

def background_action
  session[:job_key] = MiddleMan.new_worker(:class => :example_worker, :args => {:arg => params[:some_arg]})
end

def callback
  @result = MiddleMan.worker(session[:job_key]).results[:some_string]
end