if encoding_override is not None and encoding_override != 'none':

return encoding_override

if sys.stdout.isatty():

return sys.stdout.encoding

else:

if data is not None:

if not isinstance(data, str) and not isinstance(data, str):

data = str(data)

return re.search(regular_expression, data) is not None

else:

# TODO Alpha implementation, need to provide multiple interpolation methods, and add tests

if not l:

return None

k = p*(len(l) - 1)

f = math.floor(k)

c = math.ceil(k)

if c == f:

return l[int(k)]

def __init__(self):

self.values = []

self.p = None

def step(self,value,p):

if self.p is None:

self.p = p

self.values.append(value)

def finalize(self):

if len(self.values) == 0 or (self.p < 0 or self.p > 1):

return None

else:

def __init__(self, show_sql=SHOW_SQL):

self.show_sql = show_sql

self.conn = sqlite3.connect(':memory:')

self.last_temp_table_id = 10000

self.cursor = self.conn.cursor()

self.type_names = {

str: 'TEXT', int: 'INT', int : 'INT' , float: 'FLOAT', None: 'TEXT'}

self.numeric_column_types = set([int, int, float])

self.add_user_functions()

def done(self):

self.conn.commit()

def store_db_to_disk_standard(self,sqlite_db_filename,table_names_mapping):

new_db = sqlite3.connect(sqlite_db_filename,isolation_level=None)

c = new_db.cursor()

for s in self.conn.iterdump():

c.execute(s)

results = c.fetchall()

#print "executed %s results %s " % (s,results)

for source_filename_str,tn in table_names_mapping.items():

c.execute('alter table `%s` rename to `%s`' % (tn, source_filename_str))

new_db.close()

def store_db_to_disk_fast(self,sqlite_db_filename,table_names_mapping):

try:

import sqlitebck

except ImportError as e:

msg = "sqlitebck python module cannot be found - fast store to disk cannot be performed. Note that for now, sqlitebck is not packaged as part of q. In order to use the fast method, you need to manually `pip install sqlitebck` into your python environment. We obviously consider this as a bug and it will be fixed once proper packaging will be done, making the fast method the standard one."

raise MissingSqliteBckModuleException(msg)

new_db = sqlite3.connect(sqlite_db_filename)

sqlitebck.copy(self.conn,new_db)

c = new_db.cursor()

for source_filename_str,tn in table_names_mapping.items():

c.execute('alter table `%s` rename to `%s`' % (tn, source_filename_str))

new_db.close()

def store_db_to_disk(self,sqlite_db_filename,table_names_mapping,method='standard'):

if method == 'standard':

self.store_db_to_disk_standard(sqlite_db_filename,table_names_mapping)

elif method == 'fast':

self.store_db_to_disk_fast(sqlite_db_filename,table_names_mapping)

else:

raise ValueError('Unknown store-db-to-disk method %s' % method)

def add_user_functions(self):

self.conn.create_function("regexp", 2, regexp)

self.conn.create_function("sha1", 1, sha1)

self.conn.create_aggregate("percentile",2,StrictPercentile)

def is_numeric_type(self, column_type):

return column_type in self.numeric_column_types

def update_many(self, sql, params):

try:

if self.show_sql:

print(sql, " params: " + str(params))

self.cursor.executemany(sql, params)

finally:

pass # cursor.close()

def execute_and_fetch(self, q):

try:

if self.show_sql:

print(repr(q))

self.cursor.execute(q)

if self.cursor.description is not None:

query_column_names = [c[0] for c in self.cursor.description]

else:

query_column_names = None

result = self.cursor.fetchall()

finally:

pass # cursor.close()

return Sqlite3DBResults(query_column_names,result)

def _get_as_list_str(self, l):

return ",".join(['"%s"' % x.replace('"', '""') for x in l])

def _get_col_values_as_list_str(self, col_vals, col_types):

result = []

for col_val, col_type in zip(col_vals, col_types):

if col_val == '' and col_type is not str:

col_val = "null"

else:

if col_val is not None:

if "'" in col_val:

col_val = col_val.replace("'", "''")

col_val = "'" + col_val + "'"

else:

col_val = "null"

result.append(col_val)

return ",".join(result)

def generate_insert_row(self, table_name, column_names):

col_names_str = self._get_as_list_str(column_names)

question_marks = ", ".join(["?" for i in range(0, len(column_names))])

return 'INSERT INTO %s (%s) VALUES (%s)' % (table_name, col_names_str, question_marks)

def generate_begin_transaction(self):

return "BEGIN TRANSACTION"

def generate_end_transaction(self):

return "COMMIT"

# Get a list of column names so order will be preserved (Could have used OrderedDict, but

# then we would need python 2.7)

def generate_create_table(self, table_name, column_names, column_dict):

# Convert dict from python types to db types

column_name_to_db_type = dict(

(n, self.type_names[t]) for n, t in column_dict.items())

column_defs = ','.join(['"%s" %s' % (

n.replace('"', '""'), column_name_to_db_type[n]) for n in column_names])

return 'CREATE TABLE %s (%s)' % (table_name, column_defs)

def generate_temp_table_name(self):

self.last_temp_table_id += 1

tn = "temp_table_%s" % self.last_temp_table_id

return tn

def generate_drop_table(self, table_name):

return "DROP TABLE %s" % table_name

def drop_table(self, table_name):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self, msg):

self.msg = msg

def __str(self):

def __init__(self,expected_col_count,actual_col_count):

self.expected_col_count = expected_col_count

def __init__(self,expected_col_count,actual_col_count):

self.expected_col_count = expected_col_count

def __init__(self, sql):

# Currently supports only standard SELECT statements

# Holds original SQL

self.sql = sql

# Holds sql parts

self.sql_parts = sql.split()

# Set of qtable names

self.qtable_names = set()

# Dict from qtable names to their positions in sql_parts. Value here is a *list* of positions,

# since it is possible that the same qtable_name (file) is referenced in multiple positions

# and we don't want the database table to be recreated for each

# reference

self.qtable_name_positions = {}

# Dict from qtable names to their effective (actual database) table

# names

self.qtable_name_effective_table_names = {}

self.query_column_names = None

# Go over all sql parts

idx = 0

while idx < len(self.sql_parts):

# Get the part string

part = self.sql_parts[idx]

# If it's a FROM or a JOIN

if part.upper() in ['FROM', 'JOIN']:

# and there is nothing after it,

if idx == len(self.sql_parts) - 1:

# Just fail

raise Exception(

'FROM/JOIN is missing a table name after it')

qtable_name = self.sql_parts[idx + 1]

# Otherwise, the next part contains the qtable name. In most cases the next part will be only the qtable name.

# We handle one special case here, where this is a subquery as a column: "SELECT (SELECT ... FROM qtable),100 FROM ...".

# In that case, there will be an ending paranthesis as part of the name, and we want to handle this case gracefully.

# This is obviously a hack of a hack :) Just until we have

# complete parsing capabilities

if ')' in qtable_name:

leftover = qtable_name[qtable_name.index(')'):]

self.sql_parts.insert(idx + 2, leftover)

qtable_name = qtable_name[:qtable_name.index(')')]

self.sql_parts[idx + 1] = qtable_name

self.qtable_names.add(qtable_name)

if qtable_name not in list(self.qtable_name_positions.keys()):

self.qtable_name_positions[qtable_name] = []

self.qtable_name_positions[qtable_name].append(idx + 1)

idx += 2

else:

idx += 1

def set_effective_table_name(self, qtable_name, effective_table_name):

if qtable_name not in self.qtable_names:

raise Exception("Unknown qtable %s" % qtable_name)

if qtable_name in list(self.qtable_name_effective_table_names.keys()):

raise Exception(

"Already set effective table name for qtable %s" % qtable_name)

self.qtable_name_effective_table_names[

qtable_name] = effective_table_name

def get_effective_sql(self,original_names=False):

if len([x for x in self.qtable_name_effective_table_names if x is None]) != 0:

raise Exception('There are qtables without effective tables')

effective_sql = [x for x in self.sql_parts]

for qtable_name, positions in self.qtable_name_positions.items():

for pos in positions:

if not original_names:

effective_sql[pos] = self.qtable_name_effective_table_names[

qtable_name]

else:

effective_sql[pos] = "`%s`" % qtable_name

return " ".join(effective_sql)

def get_qtable_name_effective_table_names(self):

return self.qtable_name_effective_table_names

def execute_and_fetch(self, db):

db_results_obj = db.execute_and_fetch(self.get_effective_sql())

def __init__(self, delimiter, expected_column_count):

self.delimiter = delimiter

self.expected_column_count = expected_column_count

if delimiter is not None:

escaped_delimiter = re.escape(delimiter)

self.split_regexp = re.compile('(?:%s)+' % escaped_delimiter)

else:

self.split_regexp = re.compile(r'\s+')

def split(self, line):

if line and line[-1] == '\n':

line = line[:-1]

def __init__(self, mode, expected_column_count, input_delimiter, skip_header=False,disable_column_type_detection=False):

self.inferred = False

self.mode = mode

self.rows = []

self.skip_header = skip_header

self.header_row = None

self.expected_column_count = expected_column_count

self.input_delimiter = input_delimiter

self.disable_column_type_detection = disable_column_type_detection

def analyze(self, col_vals):

if self.inferred:

raise Exception("Already inferred columns")

if self.skip_header and self.header_row is None:

self.header_row = col_vals

else:

self.rows.append(col_vals)

if len(self.rows) < 100:

return False

self.do_analysis()

return True

def force_analysis(self):

# This method is called whenever there is no more data, and an analysis needs

# to be performed immediately, regardless of the amount of sample data that has

# been collected

self.do_analysis()

def determine_type_of_value(self, value):

if self.disable_column_type_detection:

return str

if value is not None:

value = value.strip()

if value == '' or value is None:

return None

try:

i = int(value)

if type(i) == int:

return int

else:

return int

except:

pass

try:

f = float(value)

return float

except:

pass

return str

def determine_type_of_value_list(self, value_list):

type_list = [self.determine_type_of_value(v) for v in value_list]

all_types = set(type_list)

if len(set(type_list)) == 1:

# all the sample lines are of the same type

return type_list[0]

else:

# check for the number of types without nulls,

type_list_without_nulls = [x for x in type_list if x is not None]

# If all the sample lines are of the same type,

if len(set(type_list_without_nulls)) == 1:

# return it

return type_list_without_nulls[0]

else:

return str

def do_analysis(self):

if self.mode == 'strict':

self._do_strict_analysis()

elif self.mode in ['relaxed', 'fluffy']:

self._do_relaxed_analysis()

else:

raise Exception('Unknown parsing mode %s' % self.mode)

if self.column_count == 1 and self.expected_column_count != 1:

print("Warning: column count is one - did you provide the correct delimiter?", file=sys.stderr)

self.infer_column_types()

self.infer_column_names()

def validate_column_names(self, value_list):

column_name_errors = []

for v in value_list:

if v is None:

# we allow column names to be None, in relaxed mode it'll be filled with default names.

# RLRL

continue

if ',' in v:

column_name_errors.append(

(v, "Column name cannot contain commas"))

continue

if self.input_delimiter in v:

column_name_errors.append(

(v, "Column name cannot contain the input delimiter. Please make sure you've set the correct delimiter"))

continue

if '\n' in v:

column_name_errors.append(

(v, "Column name cannot contain newline"))

continue

if v != v.strip():

column_name_errors.append(

(v, "Column name contains leading/trailing spaces"))

continue

try:

v.encode("utf-8", "strict").decode("utf-8")

except:

column_name_errors.append(

(v, "Column name must be UTF-8 Compatible"))

continue

# We're checking for column duplication for each field in order to be able to still provide it along with other errors

if len([x for x in value_list if x == v]) > 1:

entry = (v, "Column name is duplicated")

# Don't duplicate the error report itself

if entry not in column_name_errors:

column_name_errors.append(entry)

continue

nul_index = v.find("\x00")

if nul_index >= 0:

column_name_errors.append(

(v, "Column name cannot contain NUL"))

continue

t = self.determine_type_of_value(v)

if t != str:

column_name_errors.append((v, "Column name must be a string"))

return column_name_errors

def infer_column_names(self):

if self.header_row is not None:

column_name_errors = self.validate_column_names(self.header_row)

if len(column_name_errors) > 0:

raise BadHeaderException("Header must contain only strings and not numbers or empty strings: '%s'\n%s" % (

",".join(self.header_row), "\n".join(["'%s': %s" % (x, y) for x, y in column_name_errors])))

# use header row in order to name columns

if len(self.header_row) < self.column_count:

if self.mode == 'strict':

raise ColumnCountMismatchException("Strict mode. Header row contains less columns than expected column count(%s vs %s)" % (

len(self.header_row), self.column_count))

elif self.mode in ['relaxed', 'fluffy']:

# in relaxed mode, add columns to fill the missing ones

self.header_row = self.header_row + \

['c%s' % (x + len(self.header_row) + 1)

for x in range(self.column_count - len(self.header_row))]

elif len(self.header_row) > self.column_count:

if self.mode == 'strict':

raise ColumnCountMismatchException("Strict mode. Header row contains more columns than expected column count (%s vs %s)" % (

len(self.header_row), self.column_count))

elif self.mode in ['relaxed', 'fluffy']:

# In relaxed mode, just cut the extra column names

self.header_row = self.header_row[:self.column_count]

self.column_names = self.header_row

else:

# Column names are cX starting from 1

self.column_names = ['c%s' % (i + 1)

for i in range(self.column_count)]

def _do_relaxed_analysis(self):

column_count_list = [len(col_vals) for col_vals in self.rows]

if len(self.rows) == 0:

self.column_count = 0

else:

if self.expected_column_count is not None:

self.column_count = self.expected_column_count

else:

# If not specified, we'll take the largest row in the sample rows

self.column_count = max(column_count_list)

def get_column_count_summary(self, column_count_list):

counts = {}

for column_count in column_count_list:

counts[column_count] = counts.get(column_count, 0) + 1

return ", ".join(["%s rows with %s columns" % (v, k) for k, v in counts.items()])

def _do_strict_analysis(self):

column_count_list = [len(col_vals) for col_vals in self.rows]

if len(set(column_count_list)) != 1:

raise ColumnCountMismatchException('Strict mode. Column Count is expected to identical. Multiple column counts exist at the first part of the file. Try to check your delimiter, or change to relaxed mode. Details: %s' % (

self.get_column_count_summary(column_count_list)))

self.column_count = len(self.rows[0])

if self.expected_column_count is not None and self.column_count != self.expected_column_count:

raise ColumnCountMismatchException('Strict mode. Column count is expected to be %s but is %s' % (

self.expected_column_count, self.column_count))

self.infer_column_types()

def infer_column_types(self):

self.column_types = []

self.column_types2 = []

for column_number in range(self.column_count):

column_value_list = [

row[column_number] if column_number < len(row) else None for row in self.rows]

column_type = self.determine_type_of_value_list(column_value_list)

self.column_types.append(column_type)

column_value_list2 = [row[column_number] if column_number < len(

row) else None for row in self.rows[1:]]

column_type2 = self.determine_type_of_value_list(

column_value_list2)

self.column_types2.append(column_type2)

comparison = [x[0] == x[1] for x in zip(self.column_types, self.column_types2)]

if False in comparison and not self.skip_header:

number_of_column_types = len(set(self.column_types))

if number_of_column_types == 1 and list(set(self.column_types))[0] == str:

print('Warning - There seems to be header line in the file, but -H has not been specified. All fields will be detected as text fields, and the header line will appear as part of the data', file=sys.stderr)

def get_column_dict(self):

return dict(list(zip(self.column_names, self.column_types)))

def get_column_count(self):

return self.column_count

def get_column_names(self):

return self.column_names

def get_column_types(self):

try:

if not is_stdin:

csv_reader = csv.reader(codecs.iterdecode(f,encoding), dialect,**kwargs)

else:

csv_reader = csv.reader(f, dialect, **kwargs)

for row in csv_reader:

yield row

except ValueError as e:

if e.message is not None and e.message.startswith('could not convert string to'):

raise CouldNotConvertStringToNumericValueException(e.message)

else:

raise CouldNotParseInputException(str(e))

except Exception as e:

if str(e).startswith("field larger than field limit"):

raise ColumnMaxLengthLimitExceededException(str(e))

elif 'universal-newline' in str(e):

raise UniversalNewlinesExistException()

else:

if filename == '-':

return 'stdin'

else:

NEW = 'NEW'

INITIALIZED = 'INITIALIZED'

ANALYZED = 'ANALYZED'

def __init__(self,filename,f,encoding,dialect,is_stdin):

self.filename = filename

self.lines_read = 0

self.f = f

self.encoding = encoding

self.dialect = dialect

self.is_stdin = is_stdin

self.skipped_bom = False

def read_file_using_csv(self):

# This is a hack for utf-8 with BOM encoding in order to skip the BOM. python's csv module

# has a bug which prevents fixing it using the proper encoding, and it has been encountered by

# multiple people.

if self.encoding == 'utf-8-sig' and self.lines_read == 0 and not self.skipped_bom:

try:

BOM = self.f.read(3)

if BOM != '\xef\xbb\xbf':

raise Exception('Value of BOM is not as expected - Value is "%s"' % str(BOM))

except Exception as e:

raise Exception('Tried to skip BOM for "utf-8-sig" encoding and failed. Error message is ' + str(e))

csv_reader = encoded_csv_reader(self.encoding, self.f, dialect=self.dialect,is_stdin=self.is_stdin)

try:

for col_vals in csv_reader:

self.lines_read += 1

yield col_vals

except ColumnMaxLengthLimitExceededException as e:

msg = "Column length is larger than the maximum. Offending file is '%s' - Line is %s, counting from 1 (encoding %s). The line number is the raw line number of the file, ignoring whether there's a header or not" % (self.filename,self.lines_read + 1,self.encoding)

raise ColumnMaxLengthLimitExceededException(msg)

except UniversalNewlinesExistException as e2:

# No need to translate the exception, but we want it to be explicitly defined here for clarity

raise UniversalNewlinesExistException()

def close(self):

if self.f != sys.stdin:

def __init__(self, db, filenames_str, line_splitter, skip_header=False, gzipped=False, with_universal_newlines=False, encoding='UTF-8', mode='fluffy', expected_column_count=None, input_delimiter=None,disable_column_type_detection=False,

stdin_file=None,stdin_filename='-'):

self.db = db

self.filenames_str = filenames_str

self.skip_header = skip_header

self.gzipped = gzipped

self.table_created = False

self.line_splitter = line_splitter

self.encoding = encoding

self.mode = mode

self.expected_column_count = expected_column_count

self.input_delimiter = input_delimiter

self.stdin_file = stdin_file

self.stdin_filename = stdin_filename

self.with_universal_newlines = with_universal_newlines

self.column_inferer = TableColumnInferer(

mode, expected_column_count, input_delimiter, skip_header,disable_column_type_detection)

# Filled only after table population since we're inferring the table

# creation data

self.table_name = None

self.pre_creation_rows = []

self.buffered_inserts = []

self.effective_column_names = None

# Column type indices for columns that contain numeric types. Lazily initialized

# so column inferer can do its work before this information is needed

self.numeric_column_indices = None

self.materialized_file_list = self.materialize_file_list()

self.materialized_file_dict = {}

self.state = TableCreatorState.NEW

def materialize_file_list(self):

materialized_file_list = []

# Get the list of filenames

filenames = self.filenames_str.split("+")

# for each filename (or pattern)

for fileglob in filenames:

# Allow either stdin or a glob match

if fileglob == self.stdin_filename:

materialized_file_list.append(self.stdin_filename)

else:

materialized_file_list += glob.glob(fileglob)

# If there are no files to go over,

if len(materialized_file_list) == 0:

raise FileNotFoundException(

"No files matching '%s' have been found" % self.filenames_str)

return materialized_file_list

def get_table_name(self):

return self.table_name

def open_file(self,filename,encoding):

# TODO Support universal newlines for gzipped and stdin data as well

# Check if it's standard input or a file

if filename == self.stdin_filename:

if self.stdin_file is None:

raise UnprovidedStdInException()

f = self.stdin_file.buffer

if self.gzipped:

raise CannotUnzipStdInException()

else:

if self.gzipped or filename.endswith('.gz'):

f = gzip.GzipFile(fileobj=open(filename,'rb'))

else:

if self.with_universal_newlines:

file_opening_mode = 'rbU'

else:

file_opening_mode = 'rb'

f = open(filename,file_opening_mode)

return f

def _pre_populate(self,dialect):

# For each match

for filename in self.materialized_file_list:

if filename in list(self.materialized_file_dict.keys()):

continue

f = self.open_file(filename,self.encoding)

is_stdin = filename == self.stdin_filename

mfs = MaterializedFileState(filename,f,self.encoding,dialect,is_stdin)

self.materialized_file_dict[filename] = mfs

def _populate(self,dialect,stop_after_analysis=False):

total_data_lines_read = 0

# For each match

for filename in self.materialized_file_list:

mfs = self.materialized_file_dict[filename]

try:

try:

for col_vals in mfs.read_file_using_csv():

self._insert_row(col_vals)

if stop_after_analysis and self.column_inferer.inferred:

return

if mfs.lines_read == 0 and self.skip_header:

raise MissingHeaderException("Header line is expected but missing in file %s" % filename)

total_data_lines_read += mfs.lines_read - (1 if self.skip_header else 0)

except StrictModeColumnCountMismatchException as e:

raise ColumnCountMismatchException(

'Strict mode - Expected %s columns instead of %s columns in file %s row %s. Either use relaxed/fluffy modes or check your delimiter' % (

e.expected_col_count, e.actual_col_count, normalized_filename(mfs.filename), mfs.lines_read))

except FluffyModeColumnCountMismatchException as e:

raise ColumnCountMismatchException(

'Deprecated fluffy mode - Too many columns in file %s row %s (%s fields instead of %s fields). Consider moving to either relaxed or strict mode' % (

normalized_filename(mfs.filename), mfs.lines_read, e.actual_col_count, e.expected_col_count))

finally:

if not stop_after_analysis:

mfs.close()

self._flush_inserts()

if not self.table_created:

self.column_inferer.force_analysis()

self._do_create_table()

if total_data_lines_read == 0:

raise EmptyDataException()

def populate(self,dialect,stop_after_analysis=False):

if self.state == TableCreatorState.NEW:

self._pre_populate(dialect)

self.state = TableCreatorState.INITIALIZED

if self.state == TableCreatorState.INITIALIZED:

self._populate(dialect,stop_after_analysis=True)

self.state = TableCreatorState.ANALYZED

if stop_after_analysis:

return

if self.state == TableCreatorState.ANALYZED:

self._populate(dialect,stop_after_analysis=False)

self.state = TableCreatorState.FULLY_READ

return

def _flush_pre_creation_rows(self):

for i, col_vals in enumerate(self.pre_creation_rows):

if self.skip_header and i == 0:

# skip header line

continue

self._insert_row(col_vals)

self._flush_inserts()

self.pre_creation_rows = []

def _insert_row(self, col_vals):

# If table has not been created yet

if not self.table_created:

# Try to create it along with another "example" line of data

self.try_to_create_table(col_vals)

# If the table is still not created, then we don't have enough data, just

# store the data and return

if not self.table_created:

self.pre_creation_rows.append(col_vals)

return

# The table already exists, so we can just add a new row

self._insert_row_i(col_vals)

def initialize_numeric_column_indices_if_needed(self):

# Lazy initialization of numeric column indices

if self.numeric_column_indices is None:

column_types = self.column_inferer.get_column_types()

self.numeric_column_indices = [idx for idx, column_type in enumerate(

column_types) if self.db.is_numeric_type(column_type)]

def nullify_values_if_needed(self, col_vals):

new_vals = col_vals[:]

col_count = len(col_vals)

for i in self.numeric_column_indices:

if i >= col_count:

continue

v = col_vals[i]

if v == '':

new_vals[i] = None

return new_vals

def normalize_col_vals(self, col_vals):

# Make sure that numeric column indices are initializd

self.initialize_numeric_column_indices_if_needed()

col_vals = self.nullify_values_if_needed(col_vals)

expected_col_count = self.column_inferer.get_column_count()

actual_col_count = len(col_vals)

if self.mode == 'strict':

if actual_col_count != expected_col_count:

raise StrictModeColumnCountMismatchException(expected_col_count,actual_col_count)

return col_vals

# in all non strict mode, we add dummy data to missing columns

if actual_col_count < expected_col_count:

col_vals = col_vals + \

[None for x in range(expected_col_count - actual_col_count)]

# in relaxed mode, we merge all extra columns to the last column value

if self.mode == 'relaxed':

if actual_col_count > expected_col_count:

xxx = col_vals[:expected_col_count - 1] + \

[self.input_delimiter.join([v if v is not None else '' for v in

col_vals[expected_col_count - 1:]])]

return xxx

else:

return col_vals

if self.mode == 'fluffy':

if actual_col_count > expected_col_count:

raise FluffyModeColumnCountMismatchException(expected_col_count,actual_col_count)

return col_vals

raise Exception("Unidentified parsing mode %s" % self.mode)

def _insert_row_i(self, col_vals):

col_vals = self.normalize_col_vals(col_vals)

if self.effective_column_names is None:

self.effective_column_names = self.column_inferer.column_names[:len(col_vals)]

if len(self.effective_column_names) > 0: