ZTh"ddlZddlZddlZddlZddlZddlmZddlmZm Z m Z ddl m Z m Z ddlmZmZddlmZddlmZddlmZdd lmZdd lmZmZGd d eZd e eefde eeffdZ y)N)Path)OptionalTupleUnion)urljoinurlparse) ProgressFileconvert_keys_to_snake_case) ParquetConfig)!ParquetComplexTypesImportModeType)DataImportService)TimeFormatType)SiftRestConfig _RestServicec*eZdZUdZdZdZeed<eed<eed<eed<deffd Z dd e ee fd e d e d efdZded e d efdZej$ej(dddfded e ee fdedededeedeedeed efdZd e ee fd efdZxZS)ParquetUploadServicez/api/v1/data-imports:uploadz/api/v1/data-imports:urlz"/api/v0/data-imports:detect-config _rest_conf _upload_uri_url_uri_apikey rest_confct||t|j|j|_t|j|j |_t|j|j|_ y)N)r) super__init__r _base_uri UPLOAD_PATHrURL_PATHrDETECT_CONFIG_PATH_detect_config_uri)selfr __class__s `/home/www/backend.miabetepe.com/venv/lib/python3.12/site-packages/sift_py/data_import/parquet.pyrzParquetUploadService.__init__s[ 9-"4>>43C3CD > ")$..$:Q:Q"Rpathparquet_config show_progressreturnc .t||jj|jddit j d|j i}|jdk7r%td|jd|j |j } |d }|d }t|| 5} ddi} |jj|| | }|jdk7r%td|jd|jt|j|cd d d S#tjjtf$rtd|jwxYw#t$r}td |d }~wwxYw#1swYy xYw)a Uploads the Parquet file pointed to by `path` using a custom Parquet config. Args: path: The path to the Parquet file. parquet_config: The Parquet config. show_progress: Whether to show the status bar or not. Content-Encodingzapplication/octet-streamr%)urlheadersdataz3Config file upload request failed with status code . Invalid response: uploadUrl dataImportId Response missing required keys: N)disablez1Data file upload request failed with status code )_extract_parquet_footer_sessionpostrjsondumpsto_dict status_code ExceptiontextdecoderJSONDecodeErrorKeyErrorr r r) r r$r%r&response upload_info upload_urldata_import_idefr+s r"uploadzParquetUploadService.upload"s %==%%  "$>-~/E/E/GHI &    3 &EhFZFZE[[]^f^k^k]lm  B"--/K D)+6J"-n"=N$M(9 : Fa"$>G}}))*H ##s*GH\H\G]]_`h`m`m_no%T__nE! F F ,,h7 B0@A A B  D>qcBC C D F Fs1D- E,7A,F -r?r r)r r*r% parsed_urlr@rArDrCs r"upload_from_urlz$ParquetUploadService.upload_from_url]sIc]   $; ;' (9(9'::de ==%% &4&<&<&>&    3 &=h>R>R=SSUV^VcVcUde  6"--/K D"-n"=N!.AA ,,h7 6045 5 6  D>qcBC C Ds03CD#DDD D4!D//D4N asset_name time_path time_formatcomplex_types_import_moderun_namerun_idrelative_start_timec |j|} || d<|| ddd<|| ddd<| || ddd<|| d<||| d<||| d <t| } |j|| S) a Uploads the Parquet file pointed to by `path` to the specified asset. This function will automatically generate the Parquet Config using the footer. See the options below for what parameters can be overridden. Use `upload` if you need to specify a custom Parquet config. Set `time_path` to specify which column contains timestamp information and `time_format` to specify the time data format. Default is `TimeFormatType.ABSOLUTE_UNIX_NANOSECONDS`. Override `complex_types_import_mode` to specify how to import complex types (maps and list). Default is both strings and bytes. Override `run_name` to specify the name of the run to create for this data. Default is None. Override `run_id` to specify the id of the run to add this data to. Default is None. Override `relative_start_time` if a relative time format is used. Default is None. rN flat_dataset time_columnr$formatrTrQrRrS)_detect_config_flat_datasetr rF) r rNr$rOrPrQrRrSrT config_infor%s r"flat_dataset_uploadz(ParquetUploadService.flat_dataset_uploads066t< $. L!=F N#M26:?J N#M28<  *PcK ' 67L M3L /0  &.K #  $*K !&{3{{400r#c|t|\}}tj|jd}t j |dd}t j|j}|jj|j|ddi}|jdk7r%td|jd |j t|j }d |vrtd |||d d<|d S#tj j"t$f$r} td | d } ~ wwxYw)zjReturns a dictionary representing the flat dataset Parquet config detected from the file. zutf-8!DATA_TYPE_KEY_PARQUET_FLATDATASET)r,typer)gzip)r*r,r+r-z.Detect config request failed with status code r.r/Nr%z4Parquet config missing from detect config response: footer_offset)r4base64 b64encodedecoder7r8r_compressencoder5r6rr:r;r<r r=r>r?) r r$ footer_bytesr` encoded_data request_datacompressed_datar@rZrDs r"rYz0ParquetUploadService._detect_config_flat_datasetsQ'>d&C# m'' 5<==%%''oHZ\bGc&    3 &@AUAU@VVXYaYfYfXgh  64X]]_EK ; .RS^R_`a a:G $%o6+,, ,,h7 6045 5 6s D#D;(D66D;)T)__name__ __module__ __qualname__rrrr__annotations__strrrrr boolr rFrMrABSOLUTE_UNIX_NANOSECONDSr BOTHrr[dictrY __classcell__)r!s@r"rrsI/K)H=M LS.S# 9FCI9F&9F 9F  9Fv'B 'B&'B  'B\'5&N&NGhGmGm"& $-1+1+1CI+1 +1 $ +1 $E +13-+1 +1&c]+1 +1Z!-c4i0@!-T!-r#rfilenamer'ct|d5}|jdd|jd}tjd|ddd}|dd}|d k7r t d |j|dz d|j|}|t jj|t|z dz fcdddS#1swYyxYw) z1Return the Parquet footer bytes and footer offsetrbiz?B !!"% G HI I a !$vvj) RWW__X6\9JJQNN O O Os B+CC )!rar_r7rr}pathlibrtypingrrr urllib.parserrsift_py.data_import._utilsr r sift_py.data_import.configr )sift_py.data_import.parquet_complex_typesr sift_py.data_import.statusr sift_py.data_import.time_formatr sift_py.restrrrrnbytesintr4r#r"rsi ))*O4W8:5B-<B-J OeCI&6 O5;L Or#