o Gh @sUdZddlmZddlmZddlZddlmZmZddl Z ddl Z ddl Z ddl m Z ddlmZmZmZmZmZmZmZddlZddlZddlmZmZmZdd lmZmZ dd l!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(ddl)m*Z*m+Z+m,Z,m-Z-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;mZ>ddl?m@Z@ddlAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKddlLmMZMmNZNmOZOddlPmQmRZSddlTmUZUmVZVddlWmXZXddlYmZZZddl[m\Z\m]Z]ddl^m_Z_ddl`maZambZber9ddlcmdZdmeZemfZfddlgmhZhddlimjZjmkZkmlZlddlmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZudd l[mvZvd!Zwd"Zxd#d$Zydd)d*Zzd+d,Z{eUZ|dd/d0Z}d1Z~d2ed3<d4Zd2ed5<d6Zd2ed7<d8d8d9d9d:ZeBdgiZd;Zd2ed<<d=Zd2ed><ed? ejd@dAeejdBejdCdeegdDdBWdn 1swYdadAadEdFZ G   A  H     I "ddd_d`Z  a I     A dddjdkZddodpZGdqdrdrZGdsdtdtZGdudvdvZGdwdxdxeZGdydzdzeZGd{d|d|eZGd}d~d~eZGdddZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZ ddddZdddZe AddddZeddddZ AddddZdddZdddZdddZdddZdddZdddÄZdddńZdddDŽZddd˄Zddd΄ZdddЄZGdd҄d҃ZdS)zY High level interface to PyTables for reading and writing pandas data structures to disk ) annotations)suppressN)datetzinfo)dedent) TYPE_CHECKINGAnyCallableFinalLiteralcastoverload)config get_optionusing_pyarrow_string_dtype)libwriters)is_string_array) timezones)import_optional_dependency) patch_pickle)AttributeConflictWarningClosedFileErrorIncompatibilityWarningPerformanceWarningPossibleDataLossError)cache_readonly)find_stack_level) ensure_object is_bool_dtypeis_complex_dtype is_list_likeis_string_dtypeneeds_i8_conversion)CategoricalDtypeDatetimeTZDtypeExtensionDtype PeriodDtype)array_equivalent) DataFrame DatetimeIndexIndex MultiIndex PeriodIndex RangeIndexSeriesTimedeltaIndexconcatisna) Categorical DatetimeArray PeriodArray) PyTablesExprmaybe_expression) extract_array) ensure_index) ArrayManager BlockManager)stringify_path)adjoin pprint_thing)HashableIteratorSequence) TracebackType)ColFileNode) AnyArrayLike ArrayLikeAxisIntDtypeArgFilePathSelfShapenpt)Blockz0.15.2UTF-8cCst|tjr |d}|S)z(if we have bytes, decode them to unicoderO) isinstancenpbytes_decode)srU_/var/www/html/HyperOracle/crypto_trader/venv/lib/python3.10/site-packages/pandas/io/pytables.py_ensure_decodeds  rWencoding str | NonereturnstrcCs|durt}|SN)_default_encodingrXrUrUrV_ensure_encodingsr_cCst|tr t|}|S)z Ensure that an index / column name is a str (python 3); otherwise they may be np.string dtype. Non-string dtypes are passed through unchanged. https://github.com/pandas-dev/pandas/issues/13492 )rPr[namerUrUrV _ensure_strs rb scope_levelintcsV|dt|ttfrfdd|D}n t|rt|d}|dus't|r)|SdS)z Ensure that the where is a Term or a list of Term. This makes sure that we are capturing the scope of variables that are passed create the terms here with a frame_level=2 (we are 2 levels down) cs0g|]}|durt|rt|ddn|qS)Nrerc)r7Term).0termlevelrUrV s z _ensure_term..rfN)rPlisttupler7rglen)wherercrUrjrV _ensure_terms   rqz where criteria is being ignored as this version [%s] is too old (or not-defined), read the file in and write it out to a new file to upgrade (with the copy_to method) r incompatibility_doczu the [%s] attribute of the existing index is [%s] which conflicts with the new [%s], resetting the attribute to None attribute_conflict_docz your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->%s,key->%s] [items->%s] performance_docfixedtable)frutrvz; : boolean drop ALL nan rows when appending to a table dropna_docz~ : format default format writing format, if None, then put will default to 'fixed' and append will default to 'table' format_doczio.hdf dropna_tableF) validatordefault_format)rurvNcCsNtdur%ddl}|att|jjdkaWdtS1s wYtS)Nrstrict) _table_modtablesrAttributeErrorfile_FILE_OPEN_POLICY!_table_file_open_policy_is_strict)rrUrUrV_tabless   raTr~ path_or_bufFilePath | HDFStorekeyvalueDataFrame | Seriesmode complevel int | Nonecomplibappendboolformatindex min_itemsizeint | dict[str, int] | Nonedropna bool | None data_columns Literal[True] | list[str] | NoneerrorsNonec s|r f dd}n f dd}t|}t|trIt||||d }||WddS1sBwYdS||dS)z+store this object, close it if we opened itc s|j d S)N)rrrnan_reprrrrX)rstore rrrXrrrrrrrrUrVzto_hdf..c s|j d S)N)rrrrrrrXrputrrrUrVr$r)rrrN)r<rPr[HDFStore)rrrrrrrrrrrrrrrXrwrrUrrVto_hdfs   " rrrpstr | list | Nonestartstopcolumnslist[str] | Noneiterator chunksizec  Ks|dvr td|d|durt|dd}t|tr'|js"td|} d} n:t|}t|ts4td zt j |} Wn t tfyId} Ynw| sTt d |d t|f||d | } d } z9|dur| }t|dkrttd|d}|ddD] }t||stdq~|j}| j||||||| | dWStt tfyt|tstt | Wd1swYw)a" Read from the store, close it if we opened it. Retrieve pandas object stored in file, optionally based on where criteria. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- path_or_buf : str, path object, pandas.HDFStore Any valid string path is acceptable. Only supports the local file system, remote URLs and file-like objects are not supported. If you want to pass in a path object, pandas accepts any ``os.PathLike``. Alternatively, pandas accepts an open :class:`pandas.HDFStore` object. key : object, optional The group identifier in the store. Can be omitted if the HDF file contains a single pandas object. mode : {'r', 'r+', 'a'}, default 'r' Mode to use when opening the file. Ignored if path_or_buf is a :class:`pandas.HDFStore`. Default is 'r'. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. where : list, optional A list of Term (or convertible) objects. start : int, optional Row number to start selection. stop : int, optional Row number to stop selection. columns : list, optional A list of columns names to return. iterator : bool, optional Return an iterator object. chunksize : int, optional Number of rows to include in an iteration when using an iterator. **kwargs Additional keyword arguments passed to HDFStore. Returns ------- object The selected object. Return type depends on the object stored. See Also -------- DataFrame.to_hdf : Write a HDF file from a DataFrame. HDFStore : Low-level access to HDF files. Examples -------- >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z']) # doctest: +SKIP >>> df.to_hdf('./store.h5', 'data') # doctest: +SKIP >>> reread = pd.read_hdf('./store.h5') # doctest: +SKIP )rr+rzmode zG is not allowed while performing a read. Allowed modes are r, r+ and a.Nrerfz&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile z does not exist)rrTrz]Dataset(s) incompatible with Pandas data types, not table, or no datasets found in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets.)rprrrrr auto_close) ValueErrorrqrPris_openOSErrorr<r[NotImplementedErrorospathexists TypeErrorFileNotFoundErrorgroupsro_is_metadata_of _v_pathnameselect LookupErrorrrclose)rrrrrprrrrrkwargsrrrrcandidate_only_groupgroup_to_checkrUrUrVread_hdf;svO          rgrouprE parent_groupcCsN|j|jkrdS|}|jdkr%|j}||kr|jdkrdS|j}|jdksdS)zDCheck if a given group is a metadata group for a given parent_group.FremetaT)_v_depth _v_parent_v_name)rrcurrentparentrUrUrVrs   rc@seZdZUdZded<ded<    ddddZdddZeddZedddZ dddZ dddZ dddZ dd d!Z dd"d#Zdd%d&Zdd'd(Zdd*d+Zdd2d3Zddd7d8Zdd:d;Zdd=d>Zddd?d@ZddAdBZeddCdDZdddFdGZddHdIZ       dddMdNZ   dddQdRZ  dddTdUZ        dddVdWZ  X        Y X dddedfZdddgdhZ   X X           YdddkdlZ!   dddodpZ"   dddtduZ#ddwdxZ$ddd|d}Z%dddZ&dddZ' X     XddddZ(dddZ)ddZ*dddZ+   YddddZ,  X            Y XddddZ-dddZ.dddZ/dddZ0dS)raS Dict-like IO interface for storing pandas objects in PyTables. Either Fixed or Table format. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- path : str File path to HDF5 file. mode : {'a', 'w', 'r', 'r+'}, default 'a' ``'r'`` Read-only; no data can be modified. ``'w'`` Write; a new file is created (an existing file with the same name would be deleted). ``'a'`` Append; an existing file is opened for reading and writing, and if the file does not exist it is created. ``'r+'`` It is similar to ``'a'``, but the file must already exist. complevel : int, 0-9, default None Specifies a compression level for data. A value of 0 or None disables compression. complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib' Specifies the compression library to be used. These additional compressors for Blosc are supported (default if no compressor specified: 'blosc:blosclz'): {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd'}. Specifying a compression library which is not available issues a ValueError. fletcher32 : bool, default False If applying compression use the fletcher32 checksum. **kwargs These parameters will be passed to the PyTables open_file method. Examples -------- >>> bar = pd.DataFrame(np.random.randn(10, 4)) >>> store = pd.HDFStore('test.h5') >>> store['foo'] = bar # write to HDF5 >>> bar = store['foo'] # retrieve >>> store.close() **Create or load HDF5 file in-memory** When passing the `driver` option to the PyTables open_file method through **kwargs, the HDF5 file is loaded or created in-memory and will only be written when closed: >>> bar = pd.DataFrame(np.random.randn(10, 4)) >>> store = pd.HDFStore('test.h5', driver='H5FD_CORE') >>> store['foo'] = bar >>> store.close() # only now, data is written to disk z File | None_handler[_moderNFrrr fletcher32rrZrcKsd|vrtdtd}|dur ||jjvr td|jjd|dur,|dur,|jj}t||_|dur7d}||_d|_|rA|nd|_ ||_ ||_ d|_ |j d d|i|dS) Nrz-format is not a defined argument for HDFStorerzcomplib only supports z compression.rrrrU)rrfilters all_complibsdefault_complibr<_pathrr _complevel_complib _fletcher32_filtersopen)selfrrrrrrrrUrUrV__init__)s&  zHDFStore.__init__cC|jSr\rrrUrUrV __fspath__JszHDFStore.__fspath__cCs||jdus J|jjS)zreturn the root nodeN)_check_if_openrrootrrUrUrVrMsz HDFStore.rootcCrr\rrrUrUrVfilenameTzHDFStore.filenamercC ||Sr\)getrrrUrUrV __getitem__X zHDFStore.__getitem__cCs|||dSr\r)rrrrUrUrV __setitem__[szHDFStore.__setitem__cCrr\)removerrUrUrV __delitem__^rzHDFStore.__delitem__rac Cs@z||WSttfyYnwtdt|jd|d)z$allow attribute access to get stores'z' object has no attribute ')rKeyErrorrrtype__name__)rrarUrUrV __getattr__as zHDFStore.__getattr__cCs4||}|dur|j}|||ddfvrdSdS)zx check for existence of this key can match the exact pathname or the pathnm w/o the leading '/' NreTF)get_noder)rrnoderarUrUrV __contains__ks zHDFStore.__contains__rdcC t|Sr\)rorrrUrUrV__len__w zHDFStore.__len__cCst|j}t|d|dS)N File path:  )r>rr)rpstrrUrUrV__repr__zs zHDFStore.__repr__rKcCs|Sr\rUrrUrUrV __enter__~zHDFStore.__enter__exc_typetype[BaseException] | None exc_valueBaseException | None tracebackTracebackType | NonecC |dSr\)r)rrrrrUrUrV__exit__s zHDFStore.__exit__pandasinclude list[str]cCsZ|dkr dd|DS|dkr%|jdusJdd|jjddd DStd |d ) a Return a list of keys corresponding to objects stored in HDFStore. Parameters ---------- include : str, default 'pandas' When kind equals 'pandas' return pandas objects. When kind equals 'native' return native HDF5 Table objects. Returns ------- list List of ABSOLUTE path-names (e.g. have the leading '/'). Raises ------ raises ValueError if kind has an illegal value Examples -------- >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df) # doctest: +SKIP >>> store.get('data') # doctest: +SKIP >>> print(store.keys()) # doctest: +SKIP ['/data1', '/data2'] >>> store.close() # doctest: +SKIP rcSg|]}|jqSrUrrhnrUrUrVrlz!HDFStore.keys..nativeNcSrrUrrrUrUrVrls/Table) classnamez8`include` should be either 'pandas' or 'native' but is 'r)rr walk_nodesr)rrrUrUrVkeyss z HDFStore.keys Iterator[str]cCrr\)iterrrrUrUrV__iter__rzHDFStore.__iter__Iterator[tuple[str, list]]ccs |D]}|j|fVqdS)z' iterate on key->group N)rr)rgrUrUrVitemss zHDFStore.itemscKst}|j|kr)|jdvr|dvrn|dvr&|jr&td|jd|jd||_|jr0||jrE|jdkrEtj|j|j|j d|_ t rP|jrPd }t ||j |j|jfi||_d S) a9 Open the file in the specified mode Parameters ---------- mode : {'a', 'w', 'r', 'r+'}, default 'a' See HDFStore docstring or tables.open_file for info about modes **kwargs These parameters will be passed to the PyTables open_file method. )rw)rr)rzRe-opening the file [z ] with mode [z] will delete the current file!r)rzGCannot open HDF5 file, which is already opened, even in read-only mode.N)rrrrrrrFiltersrrrrr open_filer)rrrrmsgrUrUrVrs*   z HDFStore.opencCs|jdur |jd|_dS)z0 Close the PyTables file handle N)rrrrUrUrVrs   zHDFStore.closecCs|jdurdSt|jjS)zF return a boolean indicating whether the file is open NF)rrisopenrrUrUrVrs  zHDFStore.is_openfsynccCs^|jdur+|j|r-ttt|jWddS1s$wYdSdSdS)a Force all buffered modifications to be written to disk. Parameters ---------- fsync : bool (default False) call ``os.fsync()`` on the file handle to force writing to disk. Notes ----- Without ``fsync=True``, flushing may not guarantee that the OS writes to disk. With fsync, the operation will block until the OS claims the file has been written; however, other caching layers may still interfere. N)rflushrrrrfileno)rrrUrUrVrs   "zHDFStore.flushcCsVt||}|durtd|d||WdS1s$wYdS)a  Retrieve pandas object stored in file. Parameters ---------- key : str Returns ------- object Same type as object stored in file. Examples -------- >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df) # doctest: +SKIP >>> store.get('data') # doctest: +SKIP >>> store.close() # doctest: +SKIP NNo object named in the file)rrr _read_grouprrrrUrUrVrs  $z HDFStore.getrrrc st||} | durtd|dt|dd}|| fdd} t|| |j|||||d } | S) a6 Retrieve pandas object stored in file, optionally based on where criteria. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str Object being retrieved from file. where : list or None List of Term (or convertible) objects, optional. start : int or None Row number to start selection. stop : int, default None Row number to stop selection. columns : list or None A list of columns that if not None, will limit the return columns. iterator : bool or False Returns an iterator. chunksize : int or None Number or rows to include in iteration, return an iterator. auto_close : bool or False Should automatically close the store when finished. Returns ------- object Retrieved object from file. Examples -------- >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df) # doctest: +SKIP >>> store.get('data') # doctest: +SKIP >>> print(store.keys()) # doctest: +SKIP ['/data1', '/data2'] >>> store.select('/data1') # doctest: +SKIP A B 0 1 2 1 3 4 >>> store.select('/data1', where='columns == A') # doctest: +SKIP A 0 1 1 3 >>> store.close() # doctest: +SKIP Nrrrerfcsj|||dS)N)rrrprread_start_stop_whererrTrUrVfuncxszHDFStore.select..funcrpnrowsrrrrr)rrrq_create_storer infer_axes TableIteratorr+ get_result) rrrprrrrrrrr)itrUr(rVr.s( @   zHDFStore.selectrrcCs8t|dd}||}t|tstd|j|||dS)a return the selection as an Index .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection rerfz&can only read_coordinates with a tablerprr)rq get_storerrPr rread_coordinates)rrrprrtblrUrUrVselect_as_coordinatess   zHDFStore.select_as_coordinatescolumncCs,||}t|tstd|j|||dS)a~ return a single column from the table. This is generally only useful to select an indexable .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str column : str The column of interest. start : int or None, default None stop : int or None, default None Raises ------ raises KeyError if the column is not found (or key is not a valid store) raises ValueError if the column can not be extracted individually (it is part of a data block) z!can only read_column with a table)r6rr)r2rPr r read_column)rrr6rrr4rUrUrV select_columns # zHDFStore.select_columnc  stt|dd}t|ttfrt|dkr|d}t|tr)j||||||| dSt|ttfs4tdt|s.zInvalid table []zobject [z>] is not a table, and cannot be used in all select as multiplez,all tables must have exactly the same nrows!cSsg|] }t|tr|qSrU)rPr rhxrUrUrVrl,cSsh|] }|jddqSr)non_index_axesrhrxrUrUrV /r@z.HDFStore.select_as_multiple..cs*fddD}t|ddS)Ncsg|] }|jdqS)rprrrr"rC)r%r&r'rrUrVrl4sz=HDFStore.select_as_multiple..func..F)axisverify_integrity)r1 _consolidate)r%r&r'objs)rFrtblsr$rVr)1sz)HDFStore.select_as_multiple..funcr*T) coordinates)rqrPrmrnror[rrrr2 itertoolschainzipris_tablepathnamer+popr.r/)rrrpselectorrrrrrrrTr+rxr;_tblsr)r0rU)rFrrrJrVselect_as_multiplesf +        zHDFStore.select_as_multipleTr~rrrrrrrrr track_timesrcCsH|dur tdp d}||}|j||||||||| | | | | |ddS)a Store object in HDFStore. Parameters ---------- key : str value : {Series, DataFrame} format : 'fixed(f)|table(t)', default is 'fixed' Format to use when storing object in HDFStore. Value can be one of: ``'fixed'`` Fixed format. Fast writing/reading. Not-appendable, nor searchable. ``'table'`` Table format. Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data. index : bool, default True Write DataFrame index as a column. append : bool, default False This will force Table format, append the input data to the existing. data_columns : list of columns or True, default None List of columns to create as data columns, or True to use all columns. See `here `__. encoding : str, default None Provide an encoding for strings. track_times : bool, default True Parameter is propagated to 'create_table' method of 'PyTables'. If set to False it enables to have the same h5 files (same hashes) independent on creation time. dropna : bool, default False, optional Remove missing values. Examples -------- >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df) # doctest: +SKIP Nio.hdf.default_formatru) rrrrrrrrrXrrUr)r_validate_format_write_to_group)rrrrrrrrrrrrXrrUrrUrUrVrLs&8   z HDFStore.putc Cst|dd}z||}Wn?tytytyL}z%|dur,td|||}|durB|jddWYd}~dSWYd}~nd}~wwt |||r]|j jdddS|j sdtd|j |||dS) a: Remove pandas object partially by specifying the where condition Parameters ---------- key : str Node to remove or delete rows from where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection Returns ------- number of rows removed (or None if not a Table) Raises ------ raises KeyError if key is not a valid store rerfNz5trying to remove a node with a non-None where clause!T recursivez7can only remove with where on objects written as tablesr1) rqr2rAssertionError Exceptionrr _f_removecomall_nonerrOdelete)rrrprrrTerrrrUrUrVrs8      zHDFStore.removebool | list[str]rcCsl| durtd|durtd}|durtdpd}||}|j||||||||| | | | ||||ddS)a| Append to Table in file. Node must already exist and be Table format. Parameters ---------- key : str value : {Series, DataFrame} format : 'table' is the default Format to use when storing object in HDFStore. Value can be one of: ``'table'`` Table format. Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data. index : bool, default True Write DataFrame index as a column. append : bool, default True Append the input data to the existing. data_columns : list of columns, or True, default None List of columns to create as indexed data columns for on-disk queries, or True to use all columns. By default only the axes of the object are indexed. See `here `__. min_itemsize : dict of columns that specify minimum str sizes nan_rep : str to use as str nan representation chunksize : size to chunk the writing expectedrows : expected TOTAL row size of this table encoding : default None, provide an encoding for str dropna : bool, default False, optional Do not write an ALL nan row to the store settable by the option 'io.hdf.dropna_table'. Notes ----- Does *not* check if data being appended overlaps with existing data in the table, so be careful Examples -------- >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df1, format='table') # doctest: +SKIP >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=['A', 'B']) >>> store.append('data', df2) # doctest: +SKIP >>> store.close() # doctest: +SKIP A B 0 1 2 1 3 4 0 5 6 1 7 8 Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tablerVrv)raxesrrrrrrr expectedrowsrrrXr)rrrWrX)rrrrrcrrrrrrrrrdrrrXrrUrUrVrs6I   zHDFStore.appendddictc s|durtdt|tstd||vrtdttttjtt t }d} g} | D]\} durG| durDtd| } q4| q4| durkj |} | t| } t| | } | | || <|durs||}|rfdd|D}t|}|D]}||}qj||dd}| D]1\} | |kr|nd}j|d }|durfd d | Dnd}|j| |f||d |qdS) a Append to multiple tables Parameters ---------- d : a dict of table_name to table_columns, None is acceptable as the values of one node (this will get all the remaining columns) value : a pandas object selector : a string that designates the indexable table; all of its columns will be designed as data_columns, unless data_columns is passed, in which case these are used data_columns : list of columns to create as data columns, or True to use all columns dropna : if evaluates to True, drop rows from all tables if any single row in each table has all NaN. Default False. Notes ----- axes parameter is currently not accepted Nztaxes is currently not accepted as a parameter to append_to_multiple; you can create the tables independently insteadzQappend_to_multiple must have a dictionary specified as the way to split the valuez=append_to_multiple requires a selector that is in passed dictzs z.HDFStore.append_to_multiple..rrFcsi|] \}}|vr||qSrUrUrhrr)vrUrV sz/HDFStore.append_to_multiple..)rr)rrPrfrnextrsetrangendim _AXES_MAPrrextendrc differencer+sorted get_indexertakevalues intersectionlocrQreindexr)rrerrRrrcrrrF remain_key remain_valuesr;orderedorddidxs valid_indexrrdcvalfilteredrU)rmrrVappend_to_multiple7s\ &     zHDFStore.append_to_multipleoptlevelkindrYcCsBt||}|durdSt|tstd|j|||ddS)a Create a pytables index on the table. Parameters ---------- key : str columns : None, bool, or listlike[str] Indicate which columns to create an index on. * False : Do not create any indexes. * True : Create indexes on all columns. * None : Create indexes on all columns. * listlike : Create indexes on the given columns. optlevel : int or None, default None Optimization level, if None, pytables defaults to 6. kind : str or None, default None Kind of index, if None, pytables defaults to "medium". Raises ------ TypeError: raises if the node is not a table Nz1cannot create table index on a Fixed format store)rrr)rr2rPr r create_index)rrrrrrTrUrUrVcreate_table_indexs  zHDFStore.create_table_indexrmcCs<t||jdusJtdusJdd|jDS)a Return a list of all the top-level nodes. Each node returned is not a pandas storage object. Returns ------- list List of objects. Examples -------- >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df) # doctest: +SKIP >>> print(store.groups()) # doctest: +SKIP >>> store.close() # doctest: +SKIP [/data (Group) '' children := ['axis0' (Array), 'axis1' (Array), 'block0_values' (Array), 'block0_items' (Array)]] NcSsPg|]$}t|tjjs&t|jdds$t|dds$t|tjjr&|jdkr|qS) pandas_typeNrv) rPrlinkLinkgetattr_v_attrsrvr r)rhrrUrUrVrls     z#HDFStore.groups..)rrrr walk_groupsrrUrUrVrs zHDFStore.groupsr rp*Iterator[tuple[str, list[str], list[str]]]ccst||jdusJtdusJ|j|D]A}t|jdddur'qg}g}|jD]!}t|jdd}|durKt |tj j rJ| |j q0| |j q0|jd||fVqdS)a Walk the pytables group hierarchy for pandas objects. This generator will yield the group path, subgroups and pandas object names for each group. Any non-pandas PyTables objects that are not a group will be ignored. The `where` group itself is listed first (preorder), then each of its child groups (following an alphanumerical order) is also traversed, following the same procedure. Parameters ---------- where : str, default "/" Group where to start walking. Yields ------ path : str Full path to a group (without trailing '/'). groups : list Names (strings) of the groups contained in `path`. leaves : list Names (strings) of the pandas objects contained in `path`. Examples -------- >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df1, format='table') # doctest: +SKIP >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=['A', 'B']) >>> store.append('data', df2) # doctest: +SKIP >>> store.close() # doctest: +SKIP >>> for group in store.walk(): # doctest: +SKIP ... print(group) # doctest: +SKIP >>> store.close() # doctest: +SKIP Nrr )rrrrrrr _v_childrenryrPrGrouprrrrstrip)rrprrleaveschildrrUrUrVwalks&'  z HDFStore.walk Node | NonecCs~||ds d|}|jdusJtdusJz |j|j|}Wn tjjy0YdSwt|tj s=Jt ||S)z9return the node with the key or None if it does not existr N) r startswithrrrr exceptionsNoSuchNodeErrorrPrEr)rrrrUrUrVr#s  zHDFStore.get_nodeGenericFixed | TablecCs8||}|durtd|d||}||S)z.r)rrrXr^)rrmrrPrnr2rrr rcrrrXr)rrrrrrrrr new_storer;rTdatarrUrUrVcopy=s8       z HDFStore.copyc Cst|j}t|d|d}|jr~t|}t|rxg}g}|D]K}z||}|durA|t|j p5||t|p>dWq"t yJt ym}z||t|} |d| dWYd}~q"d}~ww|t d||7}|S|d7}|S|d 7}|S) a Print detailed information on the store. Returns ------- str Examples -------- >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['A', 'B']) >>> store = pd.HDFStore("store.h5", 'w') # doctest: +SKIP >>> store.put('data', df) # doctest: +SKIP >>> print(store.info()) # doctest: +SKIP >>> store.close() # doctest: +SKIP File path: store.h5 /data frame (shape->[2,2]) rrNzinvalid_HDFStore nodez[invalid_HDFStore node: r= EmptyzFile is CLOSED) r>rrrrvrror2rrPr[r\r=) rroutputlkeysrryr;rTdetaildstrrUrUrVinfoxs8     z HDFStore.infocCs|js t|jddS)Nz file is not open!)rrrrrUrUrVrszHDFStore._check_if_openrc Cs>z t|}W|Sty}z td|d|d}~ww)zvalidate / deprecate formatsz#invalid HDFStore format specified [r=N) _FORMAT_MAPlowerrr)rrrarUrUrVrWszHDFStore._validate_formatrODataFrame | Series | NonerXc Cs |durt|ttfstdtt|jdd}tt|jdd}|durZ|durHttdus2Jt|dds?t|tj j rDd}d}ntdt|trPd }nd }|dkrZ|d 7}d|vrt t d }z||} Wnt y} ztd |dt|d|| d} ~ ww| ||||dS|dur|dur|dkrt|dd} | dur| jdkrd}n%| jdkrd}n|dkrt|dd} | dur| jdkrd}n| jdkrd}ttttttd} z| |} Wnt y} ztd|dt|d|| d} ~ ww| ||||dS)z"return a suitable class to operateNz(value must be None, Series, or DataFramer table_typerv frame_table generic_tablezKcannot create a storer if the object is not existing nor a value are passedseriesframe_table)rrz=cannot properly create the storer for: [_STORER_MAP] [group->,value->z ,format->rXr series_tablerreappendable_seriesappendable_multiseriesappendable_frameappendable_multiframe)rrrrrwormz)rPr/r)rrWrrrrrvr  SeriesFixed FrameFixedrrnlevels GenericTableAppendableSeriesTableAppendableMultiSeriesTableAppendableFrameTableAppendableMultiFrameTable WORMTable) rrrrrXrpttt _STORER_MAPclsrar _TABLE_MAPrUrUrVr,s            zHDFStore._create_storercCst|ddr|dks |rdS|||}|j|||||d}|r9|jr-|jr1|dkr1|jr1td|js8|n||jsF|rFtd|j|||||| | | | | |||d t|t rg|ri|j |ddSdSdS) NemptyrvrruzCan only append to Tablesz0Compression not supported on Fixed format stores) objrcrrrrrrrdrrrrU)r) r_identify_groupr,rO is_existsrset_object_infowriterPr r)rrrrrcrrrrrrrrdrrrrXrrUrrTrUrUrVrXs>  zHDFStore._write_to_grouprrEcCs||}||Sr\)r,r-r#)rrrTrUrUrVr Ts zHDFStore._read_groupcCsN||}|jdus J|dur|s|jj|ddd}|dur%||}|S)z@Identify HDF5 group based on key, delete/create group if needed.NTrY)rr remove_node_create_nodes_and_group)rrrrrUrUrVrYs   zHDFStore._identify_groupcCsv|jdusJ|d}d}|D](}t|sq|}|ds"|d7}||7}||}|dur6|j||}|}q|S)z,Create nodes from key and return group name.Nr )rsplitroendswithr create_group)rrpathsrpnew_pathrrUrUrVrks   z HDFStore._create_nodes_and_group)rNNF)rr[rrrrrZrrZr[rr[)rr[rZr)rar[)rr[rZrrZrd)rZrK)rrrrrrrZr)r)rr[rZr)rZr)rZr)r)rr[rZrrZrrZrF)rrrZr)NNNNFNF)rr[rrrrrrNNNrr[rrrrNN)rr[r6r[rrrr)NNNNNFNF)rrrrrr) NTFNNNNNNr~TF)rr[rrrrrrrrrrrrrr[rUrrrrZr)NNTTNNNNNNNNNNr~)rr[rrrrbrrrrrrrrrrrrrr[rZr)NNF)rerfrrrZr)rr[rrrrYrZr)rZrm)r )rpr[rZr)rr[rZr)rr[rZr)rTNNNFT) rr[rrrrrrrrrZr)rr[rZr[)NNrOr~)rrrXr[rr[rZr)NTFNNNNNNFNNNr~T)rr[rrrrbrrrrrrrrrrrr[rUrrZr)rrE)rr[rrrZrE)rr[rZrE)1r __module__ __qualname____doc____annotations__rrpropertyrrrrrrrrrrrrrrrrrrrrr5r8rTrrrrrrrrr2rrrrWr,rXr rrrUrUrUrVrs  A !           *   -     ` $ + } L= k d ( ' <   ;5   ` > rc@s`eZdZUdZded<ded<ded<    ddddZdddZdddZddddZdS)r.aa Define the iteration interface on a table Parameters ---------- store : HDFStore s : the referred storer func : the function to execute the query where : the where of the query nrows : the rows to iterate on start : the passed start value (default is None) stop : the passed stop value (default is None) iterator : bool, default False Whether to use the default iterator. chunksize : the passed chunking value (default is 100000) auto_close : bool, default False Whether to automatically close the store at the end of iteration. rrrrrrTNFrrrrZrc Cs||_||_||_||_|jjr'|durd}|durd}|dur"|}t||}||_||_||_d|_ |s9| durE| dur?d} t | |_ nd|_ | |_ dS)Nr順) rrTr)rprOminr+rrrKrdrr) rrrTr)rpr+rrrrrrUrUrVrs,     zTableIterator.__init__r@ccs|j}|jdur td||jkr:t||j|j}|dd|j||}|}|dus1t|s2q |V||jks|dS)Nz*Cannot iterate until get_result is called.) rrKrrrrr)ror)rrrrrUrUrVrs   zTableIterator.__iter__cCs|jr |jdSdSr\)rrrrrUrUrVrszTableIterator.closerKcCs|jdurt|jtstd|jj|jd|_|S|r3t|jts&td|jj|j|j|j d}n|j}| |j|j |}| |S)Nz0can only use an iterator or chunksize on a table)rpz$can only read_coordinates on a tabler1) rrPrTr rr3rprKrrr)r)rrKrpresultsrUrUrVr/s    zTableIterator.get_result)NNFNF) rrrTrrrrrrrrZrrZr@rr)rKr) rrrrrrrrr/rUrUrUrVr.s   * r.c@s\eZdZUdZdZded<dZded<gdZ             dMdNddZe dOddZ e dPddZ dQddZ dPddZ dRddZdSddZe dSd d!ZdTd'd(Zd)d*Ze d+d,Ze d-d.Ze d/d0Ze d1d2ZdUd4d5ZdVdWd6d7ZdWd8d9ZdXd=d>ZdVd?d@ZdYdAdBZdWdCdDZdWdEdFZdWdGdHZdZdIdJZ dZdKdLZ!dS)[IndexCola an index column description class Parameters ---------- axis : axis which I reference values : the ndarray like converted values kind : a string description of this type typ : the pytables type pos : the position in the pytables Tris_an_indexableis_data_indexable)freqtz index_nameNrar[cnamerYrZrcCst|ts td||_||_||_||_|p||_||_||_ ||_ | |_ | |_ | |_ | |_| |_||_|dur>||t|jtsFJt|jtsNJdS)Nz`name` must be a str.)rPr[rryrtyprarrFposrrrrrvrmetadataset_pos)rraryrrrrFrrrrrrvrrrUrUrVrs(   zIndexCol.__init__rdcC|jjSr\)ritemsizerrUrUrVr.szIndexCol.itemsizecC |jdS)N_kindr`rrUrUrV kind_attr3 zIndexCol.kind_attrrcCs,||_|dur|jdur||j_dSdSdS)z,set the position of this column in the TableN)rr_v_pos)rrrUrUrVr7s zIndexCol.set_poscC@ttt|j|j|j|j|jf}dddt gd|DS)N,cSg|] \}}|d|qSz->rUrlrUrUrVrlB z%IndexCol.__repr__..)rarrFrr) rnmapr>rarrFrrjoinrNrtemprUrUrVr=s zIndexCol.__repr__otherrctfdddDS)compare 2 col itemsc3(|]}t|dt|dkVqdSr\rrrrrUrVrjJ  z"IndexCol.__eq__..)rarrFrrgrrrUr rV__eq__HzIndexCol.__eq__cCs || Sr\)rrrUrUrV__ne__OrzIndexCol.__ne__cCs"t|jdsdSt|jj|jjS)z%return whether I am an indexed columnriF)hasattrrvrrirrrrUrUrVrRs zIndexCol.is_indexedry np.ndarrayrXr3tuple[np.ndarray, np.ndarray] | tuple[Index, Index]c Cst|tjs Jt||jjdur||j}t|j }t ||||}i}t|j |d<|j dur:t|j |d<t }t|jdsIt|jtrLt}n |jdkrYd|vrYdd}z ||fi|}Wnty|d|vrrd|d<||fi|}Ynwt||j} | | fS)zV Convert the data from this selection to the appropriate pandas type. NrarMi8c[stdd|i|S)NordinalrU)r-)r?kwdsrUrUrVr{s z"IndexCol.convert..)rPrQndarrayrdtypefieldsrrrWr_maybe_convertrrr+r is_np_dtyper%r*r_set_tzr) rryrrXrval_kindrfactory new_pd_indexfinal_pd_indexrUrUrVconvertZs2     zIndexCol.convertcCr)zreturn the valuesryrrUrUrV take_datarzIndexCol.take_datacCrr\)rvrrrUrUrVattrszIndexCol.attrscCrr\rv descriptionrrUrUrVr*r(zIndexCol.descriptioncCst|j|jdS)z!return my current col descriptionN)rr*rrrUrUrVcolz IndexCol.colcCrzreturn my cython valuesr%rrUrUrVcvalueszIndexCol.cvaluesr@cCs t|jSr\)rryrrUrUrVrrzIndexCol.__iter__cCs\t|jdkr(t|tr||j}|dur*|jj|kr,tj ||j d|_dSdSdSdS)z maybe set a string col itemsize: min_itemsize can be an integer or a dict with this columns name with an integer size stringN)rr) rWrrPrfrrarrr StringColr)rrrUrUrVmaybe_set_sizes  zIndexCol.maybe_set_sizecCdSr\rUrrUrUrVvalidate_namesrzIndexCol.validate_nameshandlerAppendableTablercCs:|j|_||||||||dSr\)rv validate_col validate_attrvalidate_metadatawrite_metadataset_attr)rr5rrUrUrVvalidate_and_sets     zIndexCol.validate_and_setcCs^t|jdkr-|j}|dur-|dur|j}|j|kr*td|d|jd|jd|jSdS)z:validate this column: return the compared against itemsizer0Nz#Trying to store a string with len [z] in [z)] column but this column has a limit of [zC]! Consider using min_itemsize to preset the sizes on these columns)rWrr+rrr)rrcrUrUrVr7s zIndexCol.validate_colcCsJ|rt|j|jd}|dur!||jkr#td|d|jddSdSdS)Nzincompatible kind in col [ - r=)rr'rrr)rr existing_kindrUrUrVr8szIndexCol.validate_attrc Cs|jD]]}t||d}||ji}||}||vrT|durT||krT|dvrBt|||f}tj|tt dd||<t ||dqt d|jd|d|d|d |dus\|dur`|||<qdS) z set/update the info for this indexable with the key/value if there is a conflict raise/warn as needed N)rr stacklevelzinvalid info for [z] for [z], existing_value [z] conflicts with new value [r=) _info_fieldsr setdefaultrarrswarningswarnrrsetattrr)rrrridxexisting_valuewsrUrUrV update_infos.   zIndexCol.update_infocCs(||j}|dur|j|dSdS)z!set my state from the passed infoN)rra__dict__update)rrrGrUrUrVset_infos zIndexCol.set_infocCst|j|j|jdS)zset the kind for this columnN)rFr'rrrrUrUrVr;zIndexCol.set_attrcCsT|jdkr"|j}||j}|dur$|dur&t||ddds(tddSdSdSdS)z:validate that kind=category does not change the categoriescategoryNT strict_nan dtype_equalzEcannot append a categorical with different categories to the existing)rr read_metadatarr(r)rr5 new_metadata cur_metadatarUrUrVr9 s  zIndexCol.validate_metadatacCs"|jdur||j|jdSdS)zset the meta dataN)rr:r)rr5rUrUrVr: s zIndexCol.write_metadata) NNNNNNNNNNNNN)rar[rrYrZrrr)rrdrZrrrrZrr)ryrrXr[rr[rZrrr\r)r5r6rrrZr)rrrZr)r5r6rZr)"rrrrrrrrBrrrrrrrrrr$r&r'r*r+r.rr2r4r<r7r8rJrMr;r9r:rUrUrUrVrsd   +       1            rc@s2eZdZdZedddZdd d ZdddZdS)GenericIndexColz:an index which is not represented in the data of the tablerZrcCdSNFrUrrUrUrVr zGenericIndexCol.is_indexedryrrXr[rtuple[Index, Index]cCs,t|tjs Jt|tt|}||fS)z Convert the data from this selection to the appropriate pandas type. Parameters ---------- values : np.ndarray nan_rep : str encoding : str errors : str )rPrQrrr.ro)rryrrXrrrUrUrVr$ s zGenericIndexCol.convertrcCr3r\rUrrUrUrVr;1 rzGenericIndexCol.set_attrNr)ryrrXr[rr[rZr[r)rrrrrrr$r;rUrUrUrVrW s   rWcseZdZdZdZdZddgZ            d>d?fdd Zed@ddZ ed@ddZ d@ddZ dAddZ dBddZ dd ZedCd#d$Zed%d&ZedDd)d*ZedEd+d,Zed-d.Zed/d0Zed1d2Zed3d4ZdFd5d6ZdGd:d;ZdFd.)rarrrshape) rnrr>rarrrrerrNrrUrUrVrn s zDataCol.__repr__rrrcr)r c3r r\r rr rUrVrj} r z!DataCol.__eq__..)rarrrrrrUr rVr{ rzDataCol.__eq__rrGcCs@|dusJ|jdus Jt|\}}||_||_t||_dSr\)r_get_data_and_dtype_namer_dtype_to_kindr)rr dtype_namerUrUrVset_data s  zDataCol.set_datacCr)zreturn the datarrrUrUrVr& rzDataCol.take_dataryrCcCs|j}|j}|j}|jdkrd|jf}t|tr&|j}|j||jj d}|St |ds1t|t r8| |}|St |drE||}|St|rUtj||dd}|St|ra|||}|S|j||j d}|S)zW Get an appropriately typed and shaped pytables.Col object for values. rerrmrrre)rrrerrsizerPr3codes get_atom_datararrr%get_atom_datetime64get_atom_timedelta64r r ComplexColr"get_atom_string)rryrrreroatomrUrUrV _get_atom s.       zDataCol._get_atomcCstj||ddS)Nrrmrr1rrerrUrUrVrt zDataCol.get_atom_stringr type[Col]cCsR|dr|dd}d|d}n|drd}n |}|d}tt|S)z0return the PyTables column class for this columnuintNUIntrCperiodInt64Col)r capitalizerr)rrk4col_namekcaprUrUrVget_atom_coltype s     zDataCol.get_atom_coltypecCs|j|d|ddS)NrkrrerrrerrUrUrVrp rNzDataCol.get_atom_datacCtj|ddSNrrrrrrerUrUrVrq zDataCol.get_atom_datetime64cCrrrrrUrUrVrr rzDataCol.get_atom_timedelta64cCt|jddS)Nre)rrrrUrUrVre z DataCol.shapecCrr-rjrrUrUrVr. r/zDataCol.cvaluescCsh|r.t|j|jd}|dur|t|jkrtdt|j|jd}|dur0||jkr2tddSdSdS)zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)rr'rrmryrrbr)rrexisting_fieldsexisting_dtyperUrUrVr8 szDataCol.validate_attrrrXrcCst|tjs Jt||jjdur||j}|jdusJ|jdur.t|\}}t |}n|}|j}|j }t|tjs>Jt |j }|j } |j} |j} |dusRJt |} | dkrbt|| dd}n| dkrntj|dd}n| dkrztjd d |Dtd}Wnltytjd d |Dtd}YnXw|d kr| } |}| durtgtjd} nt| }|r| |} ||d k|tj8<tj|| | dd}nz |j| dd}Wnty|jddd}Ynwt |dkrt ||||d}|j!|fS)aR Convert the data from this selection to the appropriate pandas type. Parameters ---------- values : np.ndarray nan_rep : encoding : str errors : str Returns ------- index : listlike to become an Index data : ndarraylike to become a column N datetime64Tcoerce timedelta64m8[ns]rrcSg|]}t|qSrUr fromordinalrhrmrUrUrVrl r<z#DataCol.convert..cSrrUr fromtimestamprrUrUrVrl r<rOF) categoriesrvalidaterOr0rrXr)"rPrQrrrrrrrfrgrrWrrrrrasarrayobjectrravelr+float64r2anyastyperdcumsum_valuesr3 from_codesr_unconvert_string_arrayry)rryrrXr convertedrhrrrrrrrromaskrUrUrVr$ sj              zDataCol.convertcCsHt|j|j|jt|j|j|j|jdusJt|j|j|jdS)zset the data for this columnN)rFr'rryrdrrrbrrUrUrVr;I szDataCol.set_attr) NNNNNNNNNNNN)rar[rrYrr]rZrrrV)rrGrZr)ryrGrZrC)rr[rZrzrr[rZrCr)ryrrXr[rr[)rrrrrrrBrrrbrdrrrir& classmethodrvrtrrprqrrrer.r8r$r; __classcell__rUrUr_rVr\5 sZ                dr\c@sPeZdZdZdZdddZeddZedd d ZeddZ eddZ dS)DataIndexableColz+represent a data column that can be indexedTrZrcCstt|jjs tddS)N-cannot have non-object label DataIndexableCol)r"r+ryrrrrUrUrVr4V szDataIndexableCol.validate_namescCstj|dS)N)rrwrxrUrUrVrt[ rz DataIndexableCol.get_atom_stringrr[rCcCs|j|dS)NrkrrrUrUrVrp_ rzDataIndexableCol.get_atom_datacC tSr\rrrUrUrVrqc  z$DataIndexableCol.get_atom_datetime64cCrr\rrrUrUrVrrg rz%DataIndexableCol.get_atom_timedelta64Nrr) rrrrrr4rrtrprqrrrUrUrUrVrQ s    rc@seZdZdZdS)GenericDataIndexableColz(represent a generic pytables data columnN)rrrrrUrUrUrVrl src@s|eZdZUdZded<dZded<ded<ded <d ed <d Zd ed<  dPdQddZedRddZ edSddZ edd Z dTd!d"Z dUd#d$Z dVd%d&Zed'd(Zed)d*Zed+d,Zed-d.ZedWd/d0ZedRd1d2Zed3d4ZdUd5d6ZdUd7d8Zed9d:ZedRd;d<Zed=d>ZdXd@dAZdYdUdCdDZdRdEdFZ B B B BdZd[dJdKZdLdMZ Bd\d]dNdOZ dBS)^Fixedz represent an object in my store facilitate read/write of various types of objects this is an abstract base class Parameters ---------- parent : HDFStore group : Node The group node where the table resides. r[ pandas_kindru format_typetype[DataFrame | Series]obj_typerdrrrrFrrOrOr~rrErXrYrrZrcCsZt|ts Jt|tdusJt|tjsJt|||_||_t||_||_ dSr\) rPrrrrErrr_rXr)rrrrXrrUrUrVr s   zFixed.__init__cCs*|jddko|jddko|jddkS)Nrre )versionrrUrUrVis_old_version s*zFixed.is_old_versiontuple[int, int, int]cCsftt|jjdd}ztdd|dD}t|dkr$|d}W|SW|Sty2d}Y|Sw) zcompute and set our versionpandas_versionNcss|]}t|VqdSr\rdr>rUrUrVrj sz Fixed.version...rrA)rrr)rWrrrrnrror)rrrUrUrVr s   z Fixed.versioncCstt|jjddS)Nr)rWrrrrrUrUrVr ryzFixed.pandas_typecCs^||j}|dur,t|ttfr"ddd|D}d|d}|jdd|d S|jS) (return a pretty representation of myselfNrcSg|]}t|qSrUr>r>rUrUrVrl z"Fixed.__repr__..[r=12.12z (shape->))r-rerPrmrnrr)rrTjshaperUrUrVr s zFixed.__repr__cCst|j|j_tt|j_dS)zset my pandas type & versionN)r[rr'r_versionrrrUrUrVr szFixed.set_object_infocCst|}|Sr\r)rnew_selfrUrUrVr s z Fixed.copycCrr\)r+rrUrUrVre rz Fixed.shapecCrr\rrrrUrUrVrP r(zFixed.pathnamecCrr\)rrrrUrUrVr r(z Fixed._handlecCrr\)rrrrUrUrVr r(zFixed._filterscCrr\)rrrrUrUrVr r(zFixed._complevelcCrr\)rrrrUrUrVr r(zFixed._fletcher32cCrr\)rrrrUrUrVr' r(z Fixed.attrscCrXzset our object attributesNrUrrUrUrV set_attrs zFixed.set_attrscCrX)zget our object attributesNrUrrUrUrV get_attrs rzFixed.get_attrscCr)zreturn my storablerrrUrUrVstorable r/zFixed.storablecCrXrYrUrrUrUrVr rZzFixed.is_existscCr)Nr+)rrrrUrUrVr+ rz Fixed.nrowsLiteral[True] | NonecCs|durdSdS)z%validate against an existing storableNTrUrrUrUrVr szFixed.validateNcCrX)+are we trying to operate on an old version?NrU)rrprUrUrVvalidate_version rzFixed.validate_versioncCs|j}|dur dS|dS)zr infer the axes of my storer return a boolean indicating if we have a valid storer or not NFT)rr)rrTrUrUrVr- s zFixed.infer_axesrrrcCtd)Nz>cannot read on an abstract storer: subclasses should implementrrrprrrrUrUrVr# sz Fixed.readcKr)Nz?cannot write on an abstract storer: subclasses should implementrrrrUrUrVr sz Fixed.writecCs,t|||r|jj|jdddStd)zs support fully deleting the node in its entirety (only) - where specification must be None TrYNz#cannot delete on an abstract storer)r^r_rrrr)rrprrrUrUrVr` sz Fixed.delete)rOr~) rrrrErXrYrr[rZrr)rZrrr)rZrr)rZrr\NNNNrrrrr)rrrrrZr)!rrrrrrrOrrrrrrrrrerPrrrrr'rrrrr+rrr-r#rr`rUrUrUrVrp sj                           rc@seZdZUdZedediZddeDZgZ de d<dddZd=ddZd=ddZd=ddZd?d@d!d"Z d?dAd$d%ZdBd'd(ZdCd*d+Z d?dDd,d-Z d?dEd0d1ZdFd4d5Z dGdHd:d;ZdS)I GenericFixedza generified fixed versiondatetimer~cCsi|]\}}||qSrUrU)rhr;rmrUrUrVrn r<zGenericFixed.r attributesrZr[cCs|j|dS)N)_index_type_mapr)rrrUrUrV_class_to_alias$ szGenericFixed._class_to_aliascCst|tr|S|j|tSr\)rPr_reverse_index_maprr+)raliasrUrUrV_alias_to_class' s zGenericFixed._alias_to_classcCs|tt|dd}|tkrd dd}|}n|tkr#d dd}|}n|}i}d|vr7|d|d<|tur7t}d|vrXt|dtrL|d d|d<n|d|d<|tusXJ||fS) N index_classrcSs:tj|j|d}tj|dd}|dur|d|}|S)N)rr`UTC)r4 _simple_newryr* tz_localize tz_convert)ryrrdtaresultrUrUrVrw6 s z*GenericFixed._get_index_factory..fcSs$t|}tj||d}tj|ddS)Nrr`)r'r5rr-)ryrrrparrrUrUrVrwA srrzutf-8r) rrWrr*r-r+r0rPbytesrS)rr'rrwr!rrUrUrV_get_index_factory- s*     zGenericFixed._get_index_factoryrcCs$|durtd|durtddS)zE raise if any keywords are passed which are not-None Nzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyzucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r)rrrprUrUrV validate_read\ szGenericFixed.validate_readrcCrX)NTrUrrUrUrVrk rZzGenericFixed.is_existscCs|j|j_|j|j_dSr)rXr'rrrUrUrVro s zGenericFixed.set_attrsc CsRtt|jdd|_tt|jdd|_|jD]}t||tt|j|dqdS)retrieve our attributesrXNrr~)r_rr'rXrWrrrF)rrrUrUrVrt s  zGenericFixed.get_attrscKrr\)rrrrrUrUrVr| rzGenericFixed.writeNrrrrc Csddl}t|j|}|j}t|dd}t||jr"|d||}n=tt|dd} t|dd} | durt| dkrt| dd d}|j| d d}|St |d|ddS)NrrrFrr0r1rreFrrrTrr+)rFr)rrrr)rr_get_block_manager_axisrqrrrrr.rrwr)rrrrror1r|)rrprrr select_axisrcrr%r&axrdfs blk_itemsrydfoutrUrUrVr# s,     zBlockManagerFixed.readrc stj|fi|t|jtr|d}|j}|s |}|j|j _t |j D]\}}|dkr9|j s9t d|d||q*t|j|j _t |jD]"\}}|j|j}|jd|d|j|d|d|d|qOdS)Nr0rz/Columns index has to be unique for fixed formatrFr)rr1)r^rrP_mgrr: _as_manageris_consolidated consolidaterrr'r rc is_uniquerrroblocksr.rrxmgr_locsrry)rrrrrr7blkr9r_rUrVr s"   zBlockManagerFixed.write)rZr/r)rrrrrZr)r) rrrrrrrer#rrrUrUr_rVr- s   &r-c@seZdZdZeZdS)rrN)rrrrr)rrUrUrUrVr srcs(eZdZUdZdZdZded<ded<dZded <d Zd ed <  ddfd"d# Z e dd$d%Z dd&d'Z dd)d*Z dd+d,Ze dd.d/Zdd3d4Ze dd6d7Ze dd8d9Ze d:d;Ze dd?Ze d@dAZe ddCdDZe ddEdFZe ddGdHZe ddJdKZddMdNZdOdPZddRdSZddUdVZddYdZZdd[d\Z dd]d^Z!dd_d`Z"dddadbZ#ddcddZ$e%dedfZ& dddhdiZ' dddndoZ(e)ddqdrZ*dsdtZ+ dddwdxZ,e-dd{d|Z.ddddZ/dddZ0 ddddZ1 ddddZ2Z3S)r aa represent a table: facilitate read/write of various types of tables Attrs in Table Node ------------------- These are attributes that are store in the main table node, they are necessary to recreate these tables when read back in. index_axes : a list of tuples of the (original indexing axis and index column) non_index_axes: a list of tuples of the (original index axis and columns on a non-indexing axis) values_axes : a list of the columns which comprise the data of this table data_columns : a list of the columns that we are allowing indexing (these become single columns in values_axes) nan_rep : the string to use for nan representations for string objects levels : the names of levels metadata : the names of the metadata columns wide_tablervr[rrrezint | list[Hashable]r TrmrNr~rrrrErXrYr index_axeslist[IndexCol] | NonerB list[tuple[AxisInt, Any]] | None values_axeslist[DataCol] | Noner list | Noner dict | NonerZrc sPtj||||d|p g|_|pg|_|pg|_|pg|_| p!i|_| |_dS)Nr)r^rrErBrHrrr) rrrrXrrErBrHrrrr_rUrVr) s      zTable.__init__cCs|jddS)N_r)rrrrUrUrVtable_type_short> zTable.table_type_shortc Cs|t|jrd|jnd}d|d}d}|jr-ddd|jD}d|d}dd d|jD}|jd |d |jd |j d |j d|d|d S)rrrz,dc->[r=rcSrrUr[r>rUrUrVrlJ rz"Table.__repr__..rcSrrUr`rrUrUrVrlM r rz (typ->z,nrows->z,ncols->z ,indexers->[r) r-rorrrrrErrMr+ncols)rjdcrverjver jindex_axesrUrUrVrB s(  zTable.__repr__r=cCs"|jD] }||jkr|SqdS)zreturn the axis for cN)rcra)rr=rrUrUrVrT s  zTable.__getitem__c Cs|durdS|j|jkrtd|jd|jddD]?}t||d}t||d}||krZt|D]\}}||}||krKtd|d|d|dq1td|d|d|dqdS) z"validate against an existing tableNz'incompatible table_type with existing [r>r=)rErBrHzinvalid combination of [z] on appending data [z] vs current table [)rrrr rr\)rrr=svovrsaxoaxrUrUrVr[ s@   zTable.validatercCs t|jtS)z@the levels attribute is 1 or a list in the case of a multi-index)rPr rmrrUrUrVis_multi_index| s zTable.is_multi_indexrr tuple[DataFrame, list[Hashable]]c CsTt|jj}z|}Wnty}ztd|d}~wwt|ts&J||fS)ze validate that we can store the multi-index; reset and return the new object zBduplicate names/columns in the multi-index when storing as a tableN)r^fill_missing_namesrr  reset_indexrrPr))rrr  reset_objrarUrUrVvalidate_multiindex s zTable.validate_multiindexrdcCstdd|jDS)z-based on our axes, compute the expected nrowscSsg|]}|jjdqSrA)r.rerhrrUrUrVrl rz(Table.nrows_expected..)rQrrErrUrUrVnrows_expected szTable.nrows_expectedcCs d|jvS)zhas this table been createdrvrrrUrUrVr s zTable.is_existscCrNrvrrrrUrUrVr rzTable.storablecCr)z,return the table group (this is my storable))rrrUrUrVrv r/z Table.tablecCrr\)rvrrrUrUrVr r(z Table.dtypecCrr\r)rrUrUrVr* r(zTable.descriptionitertools.chain[IndexCol]cCst|j|jSr\)rLrMrErHrrUrUrVrc rNz Table.axescCstdd|jDS)z.the number of total columns in the values axescss|]}t|jVqdSr\)roryrrUrUrVrj szTable.ncols..)sumrHrrUrUrVrP sz Table.ncolscCrXrYrUrrUrUrV is_transposed rZzTable.is_transposedtuple[int, ...]cCs(ttdd|jDdd|jDS)z@return a tuple of my permutated axes, non_indexable at the frontcSsg|]}t|dqSrArrrUrUrVrl rz*Table.data_orientation..cSsg|]}t|jqSrU)rdrFrrUrUrVrl r<)rnrLrMrBrErrUrUrVdata_orientation s zTable.data_orientationdict[str, Any]csRdddddjD}fddjD}fddjD}t|||S)z.csg|] \}}|dfqSr\rU)rhrFry) axis_namesrUrVrl scs&g|]}|jtjvr|j|fqSrU)rarprrrrrUrVrl s )rErBrHrf)rd1d2d3rU)rkrrV queryables s  zTable.queryablescCdd|jDS)zreturn a list of my index colscSsg|]}|j|jfqSrU)rFrr_rUrUrVrl rz$Table.index_cols..rErrUrUrV index_cols r,zTable.index_colsrcCrp)zreturn a list of my values colscSrrUrjr_rUrUrVrl r z%Table.values_cols..)rHrrUrUrV values_cols rNzTable.values_colsrcCs|jj}|d|dS)z)return the metadata pathname for this keyz/meta/z/metarr!rUrUrV_get_metadata_path szTable._get_metadata_pathryrcCs0|jj||t|ddd|j|j|jddS)z Write out a metadata array to the key as a fixed-format Series. Parameters ---------- key : str values : ndarray Frrv)rrXrrN)rrrtr/rXrr)rrryrUrUrVr: s   zTable.write_metadatacCs0tt|jdd|ddur|j||SdS)z'return the meta data array for this keyrN)rrrrrtrrUrUrVrS szTable.read_metadatacCspt|j|j_||j_||j_|j|j_|j|j_|j|j_|j|j_|j |j_ |j |j_ |j |j_ dS)zset our table type & indexablesN) r[rr'rrrsrBrrrXrr rrrUrUrVr s        zTable.set_attrscCst|jddpg|_t|jddpg|_t|jddpi|_t|jdd|_tt|jdd|_tt|jdd|_ t|jd dpBg|_ d d |j D|_ d d |j D|_ dS) rrBNrrrrXrr~r cSg|]}|jr|qSrUrrrUrUrVrlr<z#Table.get_attrs..cSg|]}|js|qSrUrvrrUrUrVrlr<)rr'rBrrrr_rXrWrr  indexablesrErHrrUrUrVrszTable.get_attrscCsF|dur|jr!tddd|jD}tj|ttddSdSdS)rNrcSrrUrOr>rUrUrVrlrz*Table.validate_version..r@)rrrrrrDrErr)rrprIrUrUrVrs zTable.validate_versioncCsR|durdSt|ts dS|}|D]}|dkrq||vr&td|dqdS)z validate the min_itemsize doesn't contain items that are not in the axes this needs data_columns to be defined Nryzmin_itemsize has the key [z%] which is not an axis or data_column)rPrfror)rrqr;rUrUrVvalidate_min_itemsizes  zTable.validate_min_itemsizec sg}jjjtjjD]5\}\}}t|}|}|dur%dnd}|d}t|d} t|||| |j||d} || qt j t |fdd| fddtjj D|S) z/create/cache the indexables if they don't existNrOr)rarFrrrrvrrc st|tsJt}|vrt}t|}t|j}t|dd}t|dd}t|}|}t|dd} |||||||j | ||d } | S)Nrrarc) rarryrrrrvrrr) rPr[r\rr_maybe_adjust_namerrgrSrv) rr=klassruadj_nameryrrmdrr)base_posrdescr table_attrsrUrVrwTs0    zTable.indexables..fcsg|] \}}||qSrUrU)rhrr=)rwrUrVrlyr@z$Table.indexables..)r*rvr'r rrrrSrrrprrortrs) r _indexablesrrFrarur~rrr index_colrU)rrrrwrrrVrx1s2      %zTable.indexablesrc CsP|sdS|dur dS|dus|durdd|jD}t|ttfs&|g}i}|dur0||d<|dur8||d<|j}|D]h}t|j|d}|dur|jrx|j }|j } |j } |durc| |krc| n| |d<|durt| |krt| n| |d<|js|j drtd |jdi|q=||jd d vrtd |d |d|dq=dS)aZ Create a pytables index on the specified columns. Parameters ---------- columns : None, bool, or listlike[str] Indicate which columns to create an index on. * False : Do not create any indexes. * True : Create indexes on all columns. * None : Create indexes on all columns. * listlike : Create indexes on the given columns. optlevel : int or None, default None Optimization level, if None, pytables defaults to 6. kind : str or None, default None Kind of index, if None, pytables defaults to "medium". Raises ------ TypeError if trying to create an index on a complex-type column. Notes ----- Cannot index Time64Col or ComplexCol. Pytables must be >= 3.0. NFTcSrrU)rrrrUrUrVrlrz&Table.create_index..rrcomplexzColumns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.rrezcolumn z/ is not a data_column. In order to read column z: you must reload the dataframe into HDFStore and include z with the data_columns argument.rU)r-rcrPrnrmrvrrirrrr remove_indexrrrrrBr) rrrrkwrvr=rmr cur_optlevelcur_kindrUrUrVr}sX   zTable.create_indexrrr9list[tuple[np.ndarray, np.ndarray] | tuple[Index, Index]]c CsZt||||d}|}g}|jD]}||j|j||j|j|jd}| |q|S)a Create the axes sniffed from the table. Parameters ---------- where : ??? start : int or None, default None stop : int or None, default None Returns ------- List[Tuple[index_values, column_values]] r1r) SelectionrrcrMrr$rrXrr) rrprr selectionryrrresrUrUrV _read_axess   zTable._read_axesrcC|S)zreturn the data for this objrUrrrrUrUrV get_objectszTable.get_objectcst|sgS|d\}|j|i}|ddkr&|r&td|d||dur/t}n|dur5g}t|trPt|t|}|fdd | Dfd d |DS) zd take the input data_columns and min_itemize and create a data columns spec rrr,z"cannot use a multi-index on axis [z] with data_columns TNcs g|] }|dkr|vr|qSr%rUr:)existing_data_columnsrUrVrls z/Table.validate_data_columns..csg|]}|vr|qSrUrU)rhr=) axis_labelsrUrVrlr) rorrrrmrPrfrprtr)rrrrBrFrrU)rrrVvalidate_data_columnss.     zTable.validate_data_columnsr)rc/stts|jj}td|dtddurdgfddD|r=d}d d|jDt|j }|j }nd }|j } |j d ksIJt |j d krVtd g} |dur^d}tfdddD} j| } t| } |rt | }|j|d }tt| t|dddsttt| tt|dddr|} | | i}t| j|d<t| j|d<| | | fd} j| } | }t|| |j|j}| |_|d| | |!||g}t |}|d ksJt | d ksJ| D] } t"| d| d q|jd k}|#||| }|$|%}|&||| |j'|\}}g}t(t)||D]\}\}}t*}d}|rbt |d krb|d|vrbt+}|d}|dusbt|t,sbtd|r|rz|j'|}Wnt-t.fy} z td|d|j'd| d} ~ wwd}|pd|}!t/|!|j0||||j|j|d}"t1|!|j2}#|3|"}$t4|"j5j6}%d}&t7|"dddurt8|"j9}&d}'}(})t|"j5t:r|"j;})d}'tj|"j|"\}*}+||#|!t||$||%|&|)|'|(|+|*d },|, | ||,|d 7}q2dd|D}-t||j?|j|j|j|| ||-| |d }.t@|d r/|jA|._A|.B||r?|r?|.C||.S)!a0 Create and return the axes. Parameters ---------- axes: list or None The names or numbers of the axes to create. obj : DataFrame The object to create axes on. validate: bool, default True Whether to validate the obj against an existing object already written. nan_rep : A value to use for string column nan_rep. data_columns : List[str], True, or None, default None Specify the columns that we want to create to allow indexing on. * True : Use all available columns. * None : Use no columns. * List[str] : Use the specified columns. min_itemsize: Dict[str, int] or None, default None The min itemsize for a column in bytes. z/cannot properly create the storer for: [group->rr=Nrcr9rU)_get_axis_numberr)rrUrVrlLr<z&Table._create_axes..TcSrrUrkrrUrUrVrlQr Frrez)rcrUrVrjisz%Table._create_axes..rirPr rrzIncompatible appended table [z]with existing table [ values_block_) existing_colrrrXrrrrOr) rarryrrrrrrrrrcSrrU)rra)rhr+rUrUrVrlr) rrrXrrErBrHrrrr )DrPr)rrrrr-rErmrrrrrrorrorcrBr(rQarrayrvrCr rr_get_axis_namerrXrrFrrJr2 _reindex_axisrrrH_get_blocks_and_itemsrHr rNr\rr[ IndexErrorr_maybe_convert_for_string_atomryr{rrvrgrrarrrr$rrrrfrrr rzr)/rrcrrrrrr table_existsnew_infonew_non_index_axesrGr append_axisindexer exist_axisr axis_name new_indexnew_index_axesjrrrAr9vaxesrrCb_itemsr|rarranew_namedata_convertedr}rrrrrrrrhr+dcs new_tablerU)rcrrV _create_axes s0              "             zTable._create_axesrrc Cs~t|jtr |d}dd}|j}tt|}t|j}||}t|ri|d\} } t |  t |} |j | | dj}tt|}t|j}||}|D]} |j | g| dj}tt|}| |j| ||qK|rddt ||D} g}g}|D];}t|j}z| |\}}||||Wq{ttfy}zdd d |D}td |d |d}~ww|}|}||fS) Nr0csfddjDS)Ncsg|] }j|jqSrU)rrxrB)rhrCmgrrUrVrlr@zFTable._get_blocks_and_items..get_blk_items..)rArrUrrV get_blk_itemssz2Table._get_blocks_and_items..get_blk_itemsrrkcSs"i|] \}}t|||fqSrU)rntolist)rhbrrUrUrVrn=sz/Table._get_blocks_and_items..rcSrrUr)rhitemrUrUrVrlJrz/Table._get_blocks_and_items..z+cannot match existing table structure for [z] on appending data)rPr<r:r=r r;rmrAror+rur|rtrNrnryrQrrrrr)rrrrHrrrrAr9rFr new_labelsr=by_items new_blocks new_blk_itemsearrrrajitemsrUrUrVrsV           zTable._get_blocks_and_itemsrrc s|durt|}|dur'jr'tjtsJjD] }||vr&|d|qjD]\}}t|||fdd}q*|jdurS|jD] \}} } ||| | qGS)zprocess axes filtersNrcsjD]X}|}|}|dusJ||kr3jr$|tj}|||}j|d|S||vr[tt |j }t|}t t rLd|}|||}j|d|Sqt d|d)Nrkrezcannot find the field [z] for filtering!) _AXIS_ORDERSr _get_axisrYunionr+r r{r9rryrPr)r)fieldfiltopr axis_number axis_valuestakersryrrrUrVprocess_filteres$        z*Table.process_axes..process_filter) rmrYrPr insertrBrfilterr) rrrrrrFlabelsrrrrrUrrV process_axesTs   zTable.process_axesrrrdcCs|dur t|jd}d|d}dd|jD|d<|r6|dur$|jp#d}tj|||p-|jd }||d <|S|jdur@|j|d <|S) z:create the description of the table from the axes & valuesNi'rv)rardcSsi|]}|j|jqSrU)rrrrUrUrVrnr<z,Table.create_description..r* )rrrr)maxr`rcrrrrr)rrrrrdrerrUrUrVcreate_descriptions"    zTable.create_descriptionc Cs|||s dSt||||d}|}|jdurD|jD]"\}}}|j|||dd} ||| j |||j }q!t |S)zf select coordinates (row numbers) from a table; return the coordinates object Fr1Nrer) rr-r select_coordsrrr7rrilocryr+) rrprrrcoordsrrrrrUrUrVr3s   zTable.read_coordinatesr6cCs||s dS|durtd|jD]>}||jkrS|js'td|dt|jj |}| |j |j ||||j |j|jd}tt|d|j|ddSqtd|d ) zj return a single column from the table, generally only indexables are interesting FNz4read_column does not currently accept a where clausezcolumn [z=] can not be extracted individually; it is not data indexablerre)rarz] not found in the table)rr-rrcrarrrrvrirMrr$rrXrr/rrr)rr6rprrrr= col_valuesrUrUrVr7s,      zTable.read_column)Nr~NNNNNN)rrrrErXrYrr[rErFrBrGrHrIrrJrrKrZrr)r=r[rr)rrrZrZr)rZrc)rZrf)rZrh)rZr)rr[rZr[)rr[ryrrZrrr\r)rrYrZrr)rrrrrZrrr)TNNN)rr)rr)rr)rr)rrrZr))rrrrrdrrZrhr)r6r[rrrr)4rrrrrrrr rOrrrMrrrrYr^r`rrrvrr*rcrPrergrorrrsrtr:rSrrrrzrrxrrrrrr staticmethodrrrr3r7rrUrUr_rVr  s       !                    L W " * q  C 7 r c@s2eZdZdZdZ    d dddZdd d ZdS)rz a write-once read-many table: this format DOES NOT ALLOW appending to a table. writing is a one-time operation the data are stored in a format that allows for searching the data on disk rNrrrcCr)z[ read the indices and the indexing array, calculate offset rows and return z!WORMTable needs to implement readrrrUrUrVr#s zWORMTable.readrZrcKr)z write in a format that we can search later on (but cannot append to): write out the indices and the values using _write_array (e.g. a CArray) create an indexing table so that we can search z"WORMTable needs to implement writerrrUrUrVrszWORMTable.writerrr)rrrrrr#rrUrUrUrVrs  rc@sZeZdZdZdZ            dd ddZd!d"ddZd#ddZd$d%ddZdS)&r6(support the new appendable table formats appendableNFTrrrrrrUrZrcCs|s |jr |j|jd|j||||| | d}|jD]}|q|jsA|j|||| d}|| |d<|jj |jfi||j |j _ |jD]}| ||qI|j || ddS)Nrv)rcrrrrr)rrrrdrU)r)rrrrrrcr4rr create_tablerr'r< write_data)rrrcrrrrrrrdrrrrUrvroptionsrUrUrVrs4    zAppendableTable.writecs|jj}|j}g}|r*|jD]}t|jjdd}t|tj r)| |j dddqt |rD|d}|ddD]}||@}q8| }nd}dd |jD} t | } | dksZJ| d d |jD} d d | D} g} t| D]\} }|f|j|| | j}| ||qo|durd }tjt|||jd }||d}t|D]9} | |t| d||krdS|j|fdd | D|dur|ndfdd | DdqdS)z` we form the data into a 2-d including indexes,values,mask write chunk-by-chunk rrku1FrreNcSrrU)r.rrUrUrVrljr z.AppendableTable.write_data..cSg|]}|qSrU)r&rrUrUrVrlprc Ss,g|]}|tt|j|jdqSr) transposerQrollarangerrrrUrUrVrlqs,rrcg|]}|qSrUrUrend_istart_irUrVrlrcrrUrUrrrUrVrlr)indexesrry)rr r`rHr2rrgrPrQrrrrorrEr rereshaperrrqwrite_data_chunk)rrrr r+masksrrrlrnindexesrybvaluesrrm new_shaperowschunksrUrrVrOsP      zAppendableTable.write_datarrrlist[np.ndarray]rnpt.NDArray[np.bool_] | Noneryc Cs|D] }t|js dSq|djd}|t|kr#tj||jd}|jj}t|}t|D] \} } | ||| <q/t|D] \} }|||| |<q>|dura|j t dd} | sa|| }t|rr|j ||j dSdS)z Parameters ---------- rows : an empty memory space where we are putting the chunk indexes : an array of the indexes mask : an array of the masks values : an array of the values NrrFr)rQrrerorrr r rrrrgrvrr) rrrrryrmr+r rrrGrlrUrUrVrs*   z AppendableTable.write_data_chunkrrcCsb|dust|s4|dur|dur|j}|jj|jdd|S|dur%|j}|jj||d}|j|S|s:dS|j}t ||||d}| }t |dd }t|} | r| } t| | dkj} t| skdg} | d| krv| | | ddkr| dd| } t| D]} |t| | }|j||jd||jddd| } q|j| S) NTrYrFrrerr)ror+rrrrv remove_rowsrr-rrr/ sort_valuesdiffrmrrrrQreversedrxrq)rrprrr+rvrry sorted_serieslnrrpgrrrUrUrVr`sF       zAppendableTable.delete) NFNNNNNNFNNT) rrrrrrrUrrZrr)rrrrrZr) rrrrrrryrrZrrr) rrrrrrrrr`rUrUrUrVr6s& ; ;,r6c@sZeZdZUdZdZdZdZeZde d<e dd d Z e dd d Z    ddddZdS)rrrrrrrrZrcCs|jdjdkS)Nrre)rErFrrUrUrVrerNz"AppendableFrameTable.is_transposedrcCs|r|j}|S)zthese are written transposed)rrrUrUrVrszAppendableFrameTable.get_objectNrrrcsR|s dSj|||d}tjr$jjddini}fddtjD}t|dks:J|d}||d} g} tjD]\} } | j vrUqK|| \} }|ddkrgt | }nt | }|d}|dur||j |d d jr|}|}t | t| d dd }n|j}t | t| d dd }|}|jdkrt|tjr|d|jdf}t|tjrt|j||d d}nt|t rt|||d}n tj|g||d}tr|jjdks|j|jksJ|j|jftrt|d dr|d}| |qKt| dkr| d}nt!| dd}t"|||d}j#|||d}|S)Nr1rcs"g|] \}}|jdur|qSrArq)rhrr7rrUrVrls"z-AppendableFrameTable.read..rerr,r Tinplacerar`Fr3r4rrr+rk)rr)$rr-rrorBrrr rcrHr+r, from_tuples set_namesrerrrrrPrQrrrer) _from_arraysrrrdtypesrgrrrr1rr)rrprrrrrindsindrframesrr index_valsr.rir ryindex_cols_r:rrUrrVr#sf              zAppendableFrameTable.readrrrr)rrrrrrrrr)rrrrerrr#rUrUrUrVrs    rcsfeZdZdZdZdZdZeZe dddZ e dd d Z dfd d Z ddfdd ZZS)rrrrrrZrcCrXrYrUrrUrUrVrebrZz#AppendableSeriesTable.is_transposedrcCrr\rUrrUrUrVrfrZz AppendableSeriesTable.get_objectNc s<t|ts|jp d}||}tjd||jd|S)+we are going to write this as a frame tableryrrNrU)rPr)rato_framer^rrr)rrrrrar_rUrVrjs   zAppendableSeriesTable.writerrrr/cs|j}|dur!|r!t|jtsJ|jD] }||vr |d|qtj||||d}|r5|j|jdd|jdddf}|j dkrFd|_ |S)NrrETrry) rYrPr rmrr^r# set_indexrra)rrprrrrYrrTr_rUrVr#qs    zAppendableSeriesTable.readrrr\rr,)rrrrrrrrr/rrrerrrr#rrUrUr_rVrZs    rcs(eZdZdZdZdZfddZZS)rrrrc s^|jpd}||\}|_t|jtsJt|j}||t||_tj dd|i|S)rryrNrU) rar^r rPrmrr+rr^r)rrrranewobjrir_rUrVrs    z AppendableMultiSeriesTable.write)rrrrrrrrrUrUr_rVrs rc@sbeZdZUdZdZdZdZeZde d<e dd d Z e d d Z dddZ eddZddZdS)rz:a table that read/writes the generic pytables table formatrrrzlist[Hashable]r rZr[cCrr\)rrrUrUrVrrzGenericTable.pandas_typecCst|jddp |jSrarbrrUrUrVrryzGenericTable.storablercCsLg|_d|_g|_dd|jD|_dd|jD|_dd|jD|_dS)rNcSrurUrvrrUrUrVrlr<z*GenericTable.get_attrs..cSrwrUrvrrUrUrVrlr<cSrrUr`rrUrUrVrlr )rBrr rxrErHrrrUrUrVrs zGenericTable.get_attrsc Cs|j}|d}|durdnd}tdd|j||d}|g}t|jD]/\}}t|ts-Jt||}||}|dur=dnd}t |||g||j||d} | | q"|S)z0create the indexables from the table descriptionrNrOr)rarFrvrr)rarryrrvrr) r*rSrWrvr _v_namesrPr[rrr) rrer~rrrrrrurrUrUrVrxs.     zGenericTable.indexablescKr)Nz cannot write on an generic tablerrrUrUrVrszGenericTable.writeNrr)rrrrrrrrr)rrrrrrrrxrrUrUrUrVrs      "rcs^eZdZdZdZeZdZe dZ e dddZ dfd d Z ddfdd ZZS)rza frame with a multi-indexrrz ^level_\d+$rZr[cCrX)Nappendable_multirUrrUrUrVrMrZz*AppendableMultiFrameTable.table_type_shortNc sx|durg}n |dur|j}||\}|_t|jts J|jD] }||vr/|d|q#tjd||d|S)NTrrrU) rrr^r rPrmrr^r)rrrrrr_rUrVrs   zAppendableMultiFrameTable.writerrrcsDtj||||d}|j}|jfdd|jjD|_|S)NrEcs g|] }j|r dn|qSr\) _re_levelssearch)rhrarrUrVrls z2AppendableMultiFrameTable.read..)r^r#rr rrr )rrprrrr:r_rrVr#s  zAppendableMultiFrameTable.readrr\rr)rrrrrr)rrrrecompilerrrMrr#rrUrUr_rVrs  rrr)rFrHrr+cCs||}t|}|durt|}|dus||r!||r!|St|}|dur6t|j|dd}||sOtddg|j}|||<|jt|}|S)NF)sort) rr9equalsuniquerzslicerrr{rn)rrFrrr7slicerrUrUrVrs   rrr str | tzinfocCst|}|S)z+for a tz-aware type, return an encoded zone)r get_timezone)rzonerUrUrVr!s rrynp.ndarray | Indexrr*cCr3r\rUryrrrUrUrVr'srrcCr3r\rUrrUrUrVr.rZstr | tzinfo | Nonenp.ndarray | DatetimeIndexcCst|tr|jdus|j|ksJ|dur;t|tr!|j}|j}nd}|}t|}t||d}|d|}|S|rDt j |dd}|S)a coerce the values to a DatetimeIndex if tz is set preserve the input shape if possible Parameters ---------- values : ndarray or Index tz : str or tzinfo coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray Nr`rM8[ns]r) rPr*rrar&rrWrrrQr)ryrrrarUrUrVr3s   rac Cstt|tsJ|j}t|\}}t|}t|}t|j ds*t |j s*t |j r=t ||||t |ddt |dd|dSt|trFtdtj|dd} t|} | dkrotjd d | Dtjd }t ||dt|d S| d krt| ||}|j j} t ||d t| |d S| dvrt |||||dSt|tjr|j tksJ|dksJ|t}t |||||d S)Niurr)ryrrrrrzMultiIndex not supported here!FrrcSrrU) toordinalrrUrUrVrlrz"_convert_index..r)rr0)integerfloating)ryrrrr)rPr[rarfrgrrvrrrr#rrrr,rr"rQrint32r Time32Col_convert_string_arrayrr1rrr$) rarrXrrrrhrrur)ryrrUrUrVrYs^           rrcCs|dkr t|}|S|dkrt|}|S|dkr>ztjdd|Dtd}W|Sty=tjdd|Dtd}Y|Sw|dvrIt|}|S|d vrWt|d||d }|S|d krdt|d }|Std |)NrrrcSrrUrrrUrUrVrlr<z$_unconvert_index..rcSrrUrrrUrUrVrlr<)rfloatrr0rrrzunrecognized index type )r*r0rQrrrr)rrrXrrrUrUrVrs4    rrrGrcCs|jtkr|Sttj|}|jj}tj|dd} | dkr td| dkr(td| dks2|dks2|St |} | } || | <tj| dd} | dkr|t | j d D]+} | | } tj| dd} | dkr{t || krk|| nd | }td |d | d qPt| ||| j }|j}t|trt||p|dpd }t|pd |}|dur||}|dur||kr|}|jd|dd}|S)NFrrz+[date] is not implemented as a table columnrz>too many timezones in this block, create separate data columnsr0rrzNo.zCannot serialize the column [z2] because its data contents are not [string] but [z] object dtyperyz|Sr)rrr rQrrarr"rr2rrqrerorrrrPrfrdrrr7r)rarrrrrXrrrhr)rrrr+error_column_labelrrecirUrUrVrsP     rrcCs`t|rt|ddj||j|j}t|}t dt |}t j |d|d}|S)a Take a string-like that is object dtype and coerce to a fixed size string type. Parameters ---------- data : np.ndarray[object] encoding : str errors : str Handler for encoding errors. Returns ------- np.ndarray[fixed-length-string] FrreSr)ror/rr[encoderrrerr libwritersmax_len_string_arrayrQr)rrXrensuredrrUrUrVrs   rcCs|j}tj|td}t|r=tt|}d|}t |dt r1t |ddj j ||dj}n |j|ddjtdd}|durCd}t||||S) a* Inverse of _convert_string_array. Parameters ---------- data : np.ndarray[fixed-length-string] nan_rep : the storage repr of NaN encoding : str errors : str Handler for encoding errors. Returns ------- np.ndarray[object] Decoded data. rUrFr)rNr)rerQrrrror#r$rrPrr/r[rSrr!string_array_replace_from_nan_repr)rrrXrrerrrUrUrVrs   rr cCs6t|ts Jt|t|rt|||}||}|Sr\)rPr[r _need_convert_get_converter)ryr rXrconvrUrUrVr@s  rcs4|dkrddS|dkrfddStd|)NrcSstj|ddS)Nrr)rQrr?rUrUrVrJsz _get_converter..r0cst|ddS)Nr)rr+rrUrVrLsz invalid kind )r)rrXrrUrrVr)Hs r)cCs|dvrdSdS)N)rr0TFrUrkrUrUrVr(Ssr(r Sequence[int]cCslt|ts t|dkrtd|ddkr4|ddkr4|ddkr4td|}|r4|d}d|}|S) z Prior to 0.10.1, we named values blocks like: values_block_0 an the name values_0, adjust the given name if necessary. Parameters ---------- name : str version : Tuple[int, int, int] Returns ------- str z6Version is incorrect, expected sequence of 3 integers.rrerrzvalues_block_(\d+)values_)rPr[rorrrr)rarrlgrprUrUrVr{Ys$   r{ dtype_strcCst|}|dr d}|S|drd}|S|drd}|S|dr(d}|S|dr1d}|S|dr:d }|S|d rCd }|S|d rLd }|S|d rUd}|S|d kr]d }|Std|d)zA Find the "kind" string describing the given dtype name. )r0rr0rr)rdr{rr timedeltarrrOr~rzcannot interpret dtype of [r=)rWrr)r0rrUrUrVrgrs@         rgcCsbt|tr|j}|jjdd}|jjdvr t| d}nt|t r(|j }t|}||fS)zJ Convert the passed data into a storable form and a dtype string. rrmMr) rPr3rorrarrrQrr%r-r&)rrhrUrUrVrfs    rfc@s:eZdZdZ   ddd d Zd d ZddZddZdS)rz Carries out a selection operation on a tables.Table object. Parameters ---------- table : a Table object where : list of Terms (or convertible to) start, stop: indices to start and/or stop selection Nrvr rrrrZrcCsV||_||_||_||_d|_d|_d|_d|_t|rt t dt j |dd}|dvr}t |}|jt jkrV|j|j}}|durDd}|durL|jj}t ||||_n't|jjt jr}|jdurj||jksv|jdurz||jkrzt d||_Wdn1swY|jdur|||_|jdur|j\|_|_dSdSdS)NFr)rbooleanrz3where must have index locations >= start and < stop)rvrprr conditionrtermsrKr!rrrr"rQrrbool_r+r issubclassrrrgenerateevaluate)rrvrprrinferredrUrUrVrsF      zSelection.__init__c Csr|durdS|j}z t|||jjdWSty8}zd|}td|d|d}t||d}~ww)z'where can be a : dict,list,tuple,stringN)rorXrz- The passed where expression: a* contains an invalid variable reference all of the variable references must be a reference to an axis (e.g. 'index' or 'columns'), or a data_column The currently defined references are: z ) rvror6rX NameErrorrrrr)rrpryraqkeysrrUrUrVr8s"  zSelection.generatecCsX|jdur|jjj|j|j|jdS|jdur!|jj|jS|jjj|j|jdS)( generate the selection Nr) r4rv read_whererrrrKr3r#rrUrUrVrs  zSelection.selectcCs|j|j}}|jj}|durd}n|dkr||7}|dur!|}n|dkr)||7}|jdur<|jjj|j||ddS|jdurD|jSt ||S)r=NrT)rrr ) rrrvr+r4get_where_listrrKrQr)rrrr+rUrUrVrs"   zSelection.select_coordsr)rvr rrrrrZr)rrrrrr8rrrUrUrUrVrs - r)rXrYrZr[)rcrd) rNNFNTNNNNr~rO)rrrr[rrrr[rrrrYrrrrYrrrrrrrrrr[rXr[rZr) Nrr~NNNNFN)rrrr[rr[rprrrrrrrrrrr)rrErrErZrr\)rr)rFrHrr+rZr))rrrZrr)ryrrrrrrZr*)ryrrrrrrZr)ryrrrrrrZr) rar[rr+rXr[rr[rZr)rr[rXr[rr[rZr)rar[rrGrr)rrrXr[rr[rZr)ryrr r[rXr[rr[)rr[rXr[rr[)rr[rZr)rar[rr,rZr[)r0r[rZr[)rrG)r __future__r contextlibrrrrrrLrrtextwraprtypingrrr r r r r rDnumpyrQpandas._configrrr pandas._libsrrr#pandas._libs.librpandas._libs.tslibsrpandas.compat._optionalrpandas.compat.pickle_compatr pandas.errorsrrrrrpandas.util._decoratorsrpandas.util._exceptionsrpandas.core.dtypes.commonrrr r!r"r#pandas.core.dtypes.dtypesr$r%r&r'pandas.core.dtypes.missingr(rr)r*r+r,r-r.r/r0r1r2pandas.core.arraysr3r4r5pandas.core.commoncorecommonr^ pandas.core.computation.pytablesr6r7pandas.core.constructionr8pandas.core.indexes.apir9pandas.core.internalsr:r;pandas.io.commonr<pandas.io.formats.printingr=r>collections.abcr?r@rAtypesrBrrCrDrEpandas._typingrFrGrHrIrJrKrLrMrNrr]rWr_rbrgrqrrrrsrtrrsryrz config_prefixregister_optionis_boolis_one_of_factoryrrrrrrrr.rrWr\rrrrrr-rr rr6rrrrrrrrrrrrrrr)r(r{rgrfrrUrUrUrVs<   $         0     (             :  (p)-f$`ndi0B+  & @  I  &    !