o NiB@sdZddlmZddlmZddlmZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZddlZddlZddlmmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlm Z ej!d6ddZ"d6ddZ#ddZ$ddZ%ddZ&Gddde'Z(Gddde'Z)Gdd d e j*Z+   !d7d"d#Z,Gd$d%d%e'Z-Gd&d'd'e+Z.d(d)Z/Gd*d+d+ej0Z1Gd,d-d-ej0Z2d.d/Z3ej!d6d0d1Z4Gd2d3d3e'Z5Gd4d5d5e'Z6dS)8zTest utilities.)absolute_import)division)print_functionN)Iterator)absltest)dataset_builder) dataset_info) dataset_utils)example_parser)example_serializer)features)splits)utils) test_caseccst|}|Vt|dS)z*Context manager for a temporary directory.N) make_tmp_dir rm_tmp_dir)dirnametmprZ/home/ubuntu/.local/lib/python3.10/site-packages/tensorflow_datasets/testing/test_utils.pytmp_dir0s rcCs,|rtjj|stjj|tj|dS)zMake a temporary directory.)dir)tfiogfileexistsmakedirstempfilemkdtemprrrrr8s rcCstjj|dS)zRm temporary directory.N)rrrrmtreerrrrr?srcCs.tjj|rtjj|tjj|dS)z)Possibly deletes and recreates directory.N)rrrrr r)drrr remake_dirDsr"cCstjtjtddS)N test_data fake_examples)ospathjoinr__file__rrrrfake_examples_dirKsr)c@sveZdZdZddZddZddZejde dfd d Zddd d Z ddZ ejdddZ dddZ ddZd S)MockFsaEThis util wraps mock for the `tf.io.gfile` API. Usage: ``` fs = MockFs() with fs.mock(): fs.add_file('/path/to/file1', 'Content of file 1') assert tf.io.gfile.exists('/path/to/file1') with tf.io.gfile.GFile('/path/to/file2', 'w') as f: f.write('Content of file 2') tf.io.gfile.rename('/path/to/file1', '/path/to/file1_moved') assert fs.files == { '/path/to/file2': 'Content of file 2', '/path/to/file1_moved': 'Content of file 1', } ``` Attributes: files: Dict[str, str], mapping existing files -> file content cCsi|_d|_dSN)files_cmselfrrr__init__is zMockFs.__init__cCs||_|jSr+)contextmanagerr- __enter__r.rrrr2ms  zMockFs.__enter__cCs|j|||Sr+)r-__exit__)r/exc_type exc_value tracebackrrrr3qszMockFs.__exit__returnccs6| |VWddS1swYdS)zOpen the file.N)mockr.rrrr1ts "zMockFs.contextmanagerNcCs$|dur d|n|}||j|<dS)Nz Content of {})formatr,)r/r&contentrrradd_filezszMockFs.add_filecs.tjjtjjtfdd|jDS)Ncs2h|]}|rtj|tjjdqS)r) startswithr%r&relpathsplitsep).0pr&rr s  z)MockFs._list_directory..)rstripr%r&r?listr,)r/r&rrBr_list_directory~s zMockFs._list_directoryrccs|dr ||dd|v}|j|}|r t|d}nt|}|}|V|}Wdn1s9wY|rE|dn||j|<dS)zPatch `tf.io.gfile.GFile`.wbzutf-8N) r<r;r,rBytesIOencodeStringIOgetvaluedecode)r/r&mode is_binaryr:fobjf new_contentrrr_opens     z MockFs._openFcCsL|s||jvrtd||||jvrtd||j||j|<dS)NzCannot overwrite: {} -> {}zCannot rename unknown file: {})r,FileExistsErrorr9FileNotFoundErrorpop)r/from_to overwriterrr_renames  zMockFs._renamec s0tjjjtjdfddddjjjdS)Nrcs |jvSr+)r,rBr.rrs zMockFs.mock..cSsdSr+r)_rrrr]s)rrlistdirGFilerename) rr8patchobjectrrrFrUr\r.rr.rr8s z MockFs.mockr+)r7N)rG)F)__name__ __module__ __qualname____doc__r0r2r3 contextlibr1rr;rFrUr\r8rrrrr*Os    r*c@s(eZdZdZ       dddZdS)FeatureExpectationItemz"Test item of a FeatureExpectation.Nc CsP||_||_||_||_||_||_|s |dus|dur td||_||_dS)Nz1dtype and shape should only be set with transform) valueexpectedexpected_serializeddecodersdtypeshape ValueError raise_cls raise_msg) r/rjrkrlrmrnrorqrrrrrr0s  zFeatureExpectationItem.__init__)NNNNNNN)rdrerfrgr0rrrrrisrics:eZdZdZefddZejddZddZ Z S) SubTestCasezAdds subTest() context manager to the TestCase if supported. Note: To use this feature, make sure you call super() in setUpClass to initialize the sub stack. cstt|g|_dSr+)superrs setUpClass_sub_test_stack)cls __class__rrrus zSubTestCase.setUpClassccsX|j|d|j}|| dVWdn1s wY|jdS)N/)rvappendr'subTestrX)r/test_str sub_test_strrrr_subTests   zSubTestCase._subTestcsFt|trtj||dd}tjfdd|dddS||dS)z7Same as assertAllEqual but compatible with nested dict.T) dict_onlycs|d|dSNr)assertAllEqualxr.rrr]z2SubTestCase.assertAllEqualNested..N) isinstancedictr zip_nested map_nestedr)r/d1d2zipped_examplesrr.rassertAllEqualNesteds   z SubTestCase.assertAllEqualNested) rdrerfrg classmethodrurhr1rr __classcell__rrrxrrss rsTcs"fdd}|dur||S|S)aiExecute the decorated test in both graph mode and eager mode. This function returns a decorator intended to be applied to test methods in a `test_case.TestCase` class. Doing so will cause the contents of the test method to be executed twice - once in graph mode, and once with eager execution enabled. This allows unittests to confirm the equivalence between eager and graph execution. NOTE: This decorator can only be used when executing eagerly in the outer scope. For example, consider the following unittest: ```python class SomeTest(tfds.testing.TestCase): @tfds.testing.run_in_graph_and_eager_modes def test_foo(self): x = tf.constant([1, 2]) y = tf.constant([3, 4]) z = tf.add(x, y) self.assertAllEqual([4, 6], self.evaluate(z)) if __name__ == '__main__': tfds.testing.test_main() ``` This test validates that `tf.add()` has the same behavior when computed with eager execution enabled as it does when constructing a TensorFlow graph and executing the `z` tensor with a session. Args: func: function to be annotated. If `func` is None, this method returns a decorator the can be applied to a function. If `func` is not None this returns the decorator applied to `func`. config: An optional config_pb2.ConfigProto to use to configure the session when executing graphs. use_gpu: If True, attempt to run as many operations as possible on GPU. Returns: Returns a decorator that will run the decorated test method twice: once by constructing and executing a graph in a session and once with eager execution enabled. csfdd}|S)zDecorator for a method.c ststd|g|Ri||t7||jd|g|Ri|Wdn1s?wYWddSWddS1sWwYdS)zRun the decorated test method.zPMust be executing eagerly when using the run_in_graph_and_eager_modes decorator.)use_gpuconfigN)rexecuting_eagerlyrptearDownGraph as_defaultsetUp test_sessionr/argskwargs)rrSrrr decorateds"zBrun_in_graph_and_eager_modes..decorator..decoratedr)rSrrr)rSr decoratorsz/run_in_graph_and_eager_modes..decoratorNr)funcrrrrrrrun_in_graph_and_eager_modess0rc@ eZdZdZddZddZdS)RaggedConstantzContainer of tf.ragged.constant values. This simple wrapper forward the arguments to delay the RaggedTensor construction after `@run_in_graph_and_eager_modes` has been called. This is required to avoid incompabilities between Graph/eager. cOs||_t||_dSr+)_argsr_kwargsrrrrr0;szRaggedConstant.__init__cCstjj|ji|jSr+)rraggedconstantrrr.rrrbuild?szRaggedConstant.buildN)rdrerfrgr0rrrrrr3s rc@s(eZdZdZedddZddZdS)FeatureExpectationsTestCasez2Tests FeatureExpectations with full encode-decode.Nc Cs |d||j|Wdn1swY|d||j|Wdn1s3wY|durY|d|||Wdn1sTwYtd|i}|t|D]%\}}|t ||j |||||dWdn1swYqhdS)z.Test the given feature against the predicates.roNrnserialized_infoinner)fdicttestfeaturerorn) r assertEqualrornget_serialized_infor FeaturesDict_set_top_level enumeratestrassertFeatureTest) r/rrorntestsrrirrrr assertFeatureFs6   z)FeatureExpectationsTestCase.assertFeaturec Csntt|jd|ji}|jdura|d?|js'td |jt || |j|jt |||j dWdn1sBwYWddSWddS1sZwYdS|jdur|d||j||jWdn1swY|dt ||d|j id\}}|d}|d}|dtjd d |} || |jp|jWdn1swY|d !|jdur|jn|j} t|| } td d | Wdn1swY|d tjdd |j} ||| Wdn1swYWddSWddS1s0wYdS)z6Test that encode=>decoding of a value works correctly.rNraisez0test.raise_msg should be set with {} for test {}rm out_serializeoutrncSs|jSr+rn)srrrr]sz?FeatureExpectationsTestCase.assertFeatureTest..rocSs|dj|dSr)roassert_is_compatible_withrrrrr]r out_valuecSst|tr |S|Sr+)rrr)trrrr]s)dillloadsdumpsencode_examplerjrqrrrrpr9typeassertRaisesWithPredicateMatchfeatures_encode_decodermrlrrnest map_structurernrorrrrkr) r/rrrrorn input_value out_tensor out_numpy out_dtypesexpected_shape out_shapesrkrrrrfsn    "         $z-FeatureExpectationsTestCase.assertFeatureTestr+)rdrerfrgrrrrrrrrCs   rc Cs||}|}t|}t|}||}tjj |}| |j }t j|j|d} | | }tr=tt|} n tjjj|} t| } | | fS)zCRuns the full pipeline: encode > write > tmp files > read > decode.r)rrr ExampleSerializerr ExampleParserserialize_examplerdataDataset from_tensorsmap parse_example functoolspartialdecode_examplernextitercompatv1make_one_shot_iteratorget_nextr as_numpy) features_dictexamplermencoded_examplespecs serializerparserserialized_exampleds decode_fnrrrrrrs"       rc@sHeZdZdZedZdddejdddgZd d Zd d Z d dZ dS)DummyDatasetSharedGeneratorTest DatasetBuilder.1.0.0z2.0.0z0.0.9z0.0.8z0.0.7zv1.0.0)tfds_version_to_preparecCstj|tdtjiddS)Nr)rr)builderr supervised_keys)r DatasetInfor rrint64r.rrr_infos z!DummyDatasetSharedGenerator._infocCs8~tjtjjdtdidtjtjjdtddidgS)Nrange_name gen_kwargs)r SplitGeneratorSplitTRAINrangeTESTr/ dl_managerrrr_split_generatorss  z-DummyDatasetSharedGenerator._split_generatorsccs|D] }|d|ifVqdS)Nrr)r/rrrrr_generate_examplessz.DummyDatasetSharedGenerator._generate_examplesN) rdrerfrgrVersionVERSIONSUPPORTED_VERSIONSrrrrrrrrs   rc@s2eZdZdZedZddZddZddZ d S) DummyMnistrrcCs,tj|ttjddtjdddddS)Nrr)ro ) num_classesimagelabelzMnist description.)rr description)rrr rImage ClassLabelr.rrrrs  zDummyMnist._infocCs(tjtjjtdtjtjjtdgS)Nr)r rrrrrrrrrrszDummyMnist._split_generatorsccs4tdD]}|tjdtjd|ddfVqdS)Nrrrr r )rnponesuint8)r/rrrrrs  zDummyMnist._generate_examplesN) rdrerfrgrrrrrrrrrrrs   rcCsttjdS)zEntrypoint for tests.N)renable_v2_behaviorrmainrrrr test_mainsrc#sJdfdd }tjd| dVWddS1swYdS)zMock out the kaggle CLI. Args: err_msg: `str`, if provided, the kaggle CLI will raise a CalledProcessError and this will be the command output. Yields: None, context will have kaggle CLI mocked out. Ncs|sJ|ddks J|d}rtd|||dd}tj|d}tjj |d }| |Wdn1sAwYd ||S) z/Mock subprocess.check_output for download call.downloadrz--pathz output.txtrHNzDownloading {} to {}) subprocessCalledProcessErrorindexr%r&r'rrrr`writer9) command_argsencodingcompetition_or_datasetout_dirfpathrSerr_msgrr check_outputs  z%mock_kaggle_api..check_outputzsubprocess.check_outputr+)rr8rb)r$r%rr#rmock_kaggle_apis   "r&c@r)DummySerializerz-To mock example_serializer.ExampleSerializer.cC~dSr+rr/rrrrr0(zDummySerializer.__init__cCst|Sr+)bytes)r/rrrrr+sz!DummySerializer.serialize_exampleN)rdrerfrgr0rrrrrr'% r'c@r) DummyParserz%To mock example_parser.ExampleParser.cCr(r+rr)rrrr02r*zDummyParser.__init__cCs|Sr+r)r/exrrrr5szDummyParser.parse_exampleN)rdrerfrgr0rrrrrr-/r,r-r+)NNT)7rg __future__rrrrhrrr%rrtypingr absl.testingrrnumpyrtensorflow.compat.v2rv2rtensorflow_datasets.corerrr r r r r rtensorflow_datasets.testingrr1rrrr"r)rcr*riTestCasersrrrrGeneratorBasedBuilderrrrr&r'r-rrrrs^                ^# J]$!