>>> from meza import io, process as pr, convert as cv
>>> from io import open

>>> # Load the csv file
>>> records = io.read_csv('data.csv')

>>> # `records` are iterators over the rows
>>> row = next(records)
>>> row
{'col1': 'hello', 'col2': '5/4/82', 'col3': '1'}

>>> # Let's replace the first row so as not to lose any data
>>> records = pr.prepend(records, row)

# Guess column types. Note: `detect_types` returns a new `records`
# generator since it consumes rows during type detection
>>> records, result = pr.detect_types(records)
>>> {t['id']: t['type'] for t in result['types']}
{'col1': 'text', 'col2': 'date', 'col3': 'int'}

# Now type cast the records. Note: most `meza.process` functions return
# generators, so lets wrap the result in a list to view the data
>>> casted = list(pr.type_cast(records, result['types']))
>>> casted[0]
{'col1': 'hello', 'col2': datetime.date(1982, 5, 4), 'col3': 1}

# Cut out the first column of data and merge the rows to get the max value
# of the remaining columns. Note: since `merge` (by definition) will always
# contain just one row, it is returned as is (not wrapped in a generator)
>>> cut_recs = pr.cut(casted, ['col1'], exclude=True)
>>> merged = pr.merge(cut_recs, pred=bool, op=max)
>>> merged
{'col2': datetime.date(2015, 1, 1), 'col3': 3}

# Now write merged data back to a new csv file.
>>> io.write('out.csv', cv.records2csv(merged))

# View the result
>>> with open('out.csv', 'utf-8') as f:
...     f.read()
'col2,col3\n2015-01-01,3\n'

>>> from io import open, StringIO
>>> from meza import io

"""Read a filepath"""
>>> records = io.read_json('path/to/file.json')

"""Read a file like object and de-duplicate the header"""
>>> f = StringIO('col,col\nhello,world\n')
>>> records = io.read_csv(f, dedupe=True)

"""View the first row"""
>>> next(records)
{'col': 'hello', 'col_2': 'world'}

"""Read the 1st sheet of an xls file object opened in text mode."""
# Also, santize the header names by converting them to lowercase and
# replacing whitespace and invalid characters with `_`.
>>> with open('path/to/file.xls', 'utf-8') as f:
...     for row in io.read_xls(f, sanitize=True):
...         # do something with the `row`
...         pass

"""Read the 2nd sheet of an xlsx file object opened in binary mode"""
# Note: sheets are zero indexed
>>> with open('path/to/file.xlsx') as f:
...     records = io.read_xls(f, encoding='utf-8', sheet=1)
...     first_row = next(records)
...     # do something with the `first_row`

"""Read any recognized file"""
>>> records = io.read('path/to/file.geojson')
>>> f.seek(0)
>>> records = io.read(f, ext='csv', dedupe=True)

>>> import itertools as it
>>> import random

>>> from io import StringIO
>>> from meza import io, process as pr, convert as cv, stats

# Create some data in the same structure as what the various `read...`
# functions output
>>> header = ['A', 'B', 'C', 'D']
>>> data = [(random.random() for _ in range(4)) for x in range(7)]
>>> df = [dict(zip(header, d)) for d in data]
>>> df[0]
{'A': 0.53908..., 'B': 0.28919..., 'C': 0.03003..., 'D': 0.65363...}

"""Sort records by the value of column `B` --> df.sort_values(by='B')"""
>>> next(pr.sort(df, 'B'))
{'A': 0.53520..., 'B': 0.06763..., 'C': 0.02351..., 'D': 0.80529...}

"""Select column `A` --> df['A']"""
>>> next(pr.cut(df, ['A']))
{'A': 0.53908170489952006}

"""Select the first three rows of data --> df[0:3]"""
>>> len(list(it.islice(df, 3)))
3

"""Select all data whose value for column `A` is less than 0.5
--> df[df.A < 0.5]
"""
>>> next(pr.tfilter(df, 'A', lambda x: x < 0.5))
{'A': 0.21000..., 'B': 0.25727..., 'C': 0.39719..., 'D': 0.64157...}

# Note: since `aggregate` and `merge` (by definition) return just one row,
# they return them as is (not wrapped in a generator).
"""Calculate the mean of column `A` across all data --> df.mean()['A']"""
>>> pr.aggregate(df, 'A', stats.mean)['A']
0.5410437473067938

"""Calculate the sum of each column across all data --> df.sum()"""
>>> pr.merge(df, pred=bool, op=sum)
{'A': 3.78730..., 'C': 2.82875..., 'B': 3.14195..., 'D': 5.26330...}

>>> import itertools as it

>>> from meza import io, process as pr, convert as cv

"""Combine the files into one iterator
--> csvstack file1.csv file2.csv
"""
>>> records = io.join('file1.csv', 'file2.csv')
>>> next(records)
{'col_1': '1', 'col_2': 'dill', 'col_3': 'male'}
>>> next(it.islice(records, 4, None))
{'col_1': '6', 'col_2': 'jill', 'col_3': 'female'}

# Now let's create a persistent records list
>>> records = list(io.read_csv('file1.csv'))

"""Sort records by the value of column `col_2`
--> csvsort -c col_2 file1.csv
"""
>>> next(pr.sort(records, 'col_2'))
{'col_1': '2', 'col_2': 'bob', 'col_3': 'male'

"""Select column `col_2` --> csvcut -c col_2 file1.csv"""
>>> next(pr.cut(records, ['col_2']))
{'col_2': 'dill'}

"""Select all data whose value for column `col_2` contains `jan`
--> csvgrep -c col_2 -m jan file1.csv
"""
>>> next(pr.grep(records, [{'pattern': 'jan'}], ['col_2']))
{'col_1': '3', 'col_2': 'jane', 'col_3': 'female'}

"""Convert a csv file to json --> csvjson -i 4 file1.csv"""
>>> io.write('file.json', cv.records2json(records))

# View the result
>>> with open('file.json', 'utf-8') as f:
...     f.read()
'[{"col_1": "1", "col_2": "dill", "col_3": "male"}, {"col_1": "2",
"col_2": "bob", "col_3": "male"}, {"col_1": "3", "col_2": "jane",
"col_3": "female"}]'

>>> from meza import io, process as pr, convert as cv

# Load the geojson file and peek at the results
>>> records, peek = pr.peek(io.read_geojson('file.geojson'))
>>> peek[0]
{'lat': 20, 'type': 'Point', 'lon': 10, 'id': 11}

"""Split the records by feature ``id`` and select the first feature
--> geojsplit -k id file.geojson
"""
>>> splits = pr.split(records, 'id')
>>> feature_records, name = next(splits)
>>> name
11

"""Convert the feature records into a GeoJSON file-like object"""
>>> geojson = cv.records2geojson(feature_records)
>>> geojson.readline()
'{"type": "FeatureCollection", "bbox": [10, 20, 10, 20], "features": '
'[{"type": "Feature", "id": 11, "geometry": {"type": "Point", '
'"coordinates": [10, 20]}, "properties": {"id": 11}}], "crs": {"type": '
'"name", "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}}}'

# Note: you can also write back to a file as shown previously
# io.write('file.geojson', geojson)

>>> from meza import io, convert as cv
>>> from io import StringIO, open

# First let's create a simple tsv file like object
>>> f = StringIO('col1\tcol2\nhello\tworld\n')
>>> f.seek(0)

# Next create a records list so we can reuse it
>>> records = list(io.read_tsv(f))
>>> records[0]
{'col1': 'hello', 'col2': 'world'}

# Now we're ready to write the records data to file

"""Create a csv file like object"""
>>> cv.records2csv(records).readline()
'col1,col2\n'

"""Create a json file like object"""
>>> cv.records2json(records).readline()
'[{"col1": "hello", "col2": "world"}]'

"""Write back csv to a filepath"""
>>> io.write('file.csv', cv.records2csv(records))
>>> with open('file.csv', 'utf-8') as f_in:
...     f_in.read()
'col1,col2\nhello,world\n'

"""Write back json to a filepath"""
>>> io.write('file.json', cv.records2json(records))
>>> with open('file.json', 'utf-8') as f_in:
...     f_in.readline()
'[{"col1": "hello", "col2": "world"}]'

from meza import process as pr

# First create some records and types. Also, convert the records to a list
# so we can reuse them.
>>> records = [{'a': 'one', 'b': 2}, {'a': 'five', 'b': 10, 'c': 20.1}]
>>> records, result = pr.detect_types(records)
>>> records, types = list(records), result['types']
>>> types
[
    {'type': 'text', 'id': 'a'},
    {'type': 'int', 'id': 'b'},
    {'type': 'float', 'id': 'c'}]

>>> import pandas as pd
>>> from meza import convert as cv

"""Convert the records to a DataFrame"""
>>> df = cv.records2df(records, types)
>>> df
        a   b   c
0   one   2   NaN
1  five  10  20.1
# Alternatively, you can do `pd.DataFrame(records)`

"""Convert the DataFrame back to records"""
>>> next(cv.df2records(df))
{'a': 'one', 'b': 2, 'c': nan}

>>> import numpy as np

>>> from array import array
>>> from meza import convert as cv

"""Convert records to a structured array"""
>>> recarray = cv.records2array(records, types)
>>> recarray
rec.array([('one', 2, nan), ('five', 10, 20.100000381469727)],
          dtype=[('a', 'O'), ('b', '<i4'), ('c', '<f4')])
>>> recarray.b
array([ 2, 10], dtype=int32)

"""Convert records to a native array"""
>>> narray = cv.records2array(records, types, native=True)
>>> narray
[[array('u', 'a'), array('u', 'b'), array('u', 'c')],
[array('u', 'one'), array('u', 'five')],
array('i', [2, 10]),
array('f', [0.0, 20.100000381469727])]

"""Convert a 2-D NumPy array to a records generator"""
>>> data = np.array([[1, 2, 3], [4, 5, 6]], np.int32)
>>> data
array([[1, 2, 3],
       [4, 5, 6]], dtype=int32)
>>> next(cv.array2records(data))
{'column_1': 1, 'column_2': 2, 'column_3': 3}

"""Convert the structured array back to a records generator"""
>>> next(cv.array2records(recarray))
{'a': 'one', 'b': 2, 'c': nan}

"""Convert the native array back to records generator"""
>>> next(cv.array2records(narray, native=True))
{'a': 'one', 'b': 2, 'c': 0.0}