diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc01279 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.idea +*.pyc +.pytest_cache +.python-version +.vscode/ +__pycache__ +db.sqlite3 +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e6c7b6e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-merge-conflict + - id: check-added-large-files + - id: check-ast + - id: check-symlinks + - id: check-yaml + args: ['--unsafe'] + - id: trailing-whitespace + - id: check-json + - id: debug-statements + - id: detect-aws-credentials + args: + - --allow-missing-credentials + - id: pretty-format-json + args: + - --autofix + exclude: Pipfile.lock + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/pycqa/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + additional_dependencies: [ flake8-print ] + files: '\.py$' + args: + - --select=F401,F403,F406,F821,T001,T003 + - repo: https://github.com/PyCQA/autoflake + rev: v1.7.7 + hooks: + - id: autoflake + files: '\.py$' + exclude: '^\..*' + args: ["--in-place", "--remove-all-unused-imports"] + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + args: ["--target-version", "py39"] diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..1c36dc0 --- /dev/null +++ b/Pipfile @@ -0,0 +1,18 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pytz = "==2023.3" +pytest = "==7.4.0" +sqlalchemy = "==2.0.18" +pre-commit = "==3.2.2" +freezegun = "==1.2.2" +pandas = "==2.0.3" +pymongo = "==4.4.0" + +[dev-packages] + +[requires] +python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..8b3dbbc --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,415 @@ +{ + "_meta": { + "hash": { + "sha256": "456f4bead300eb05bd9a4ea0468f44d64e2db868e773f84f55dfe0061d6520a7" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.11" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "cfgv": { + "hashes": [ + "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426", + "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==3.3.1" + }, + "distlib": { + "hashes": [ + "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46", + "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e" + ], + "version": "==0.3.6" + }, + "dnspython": { + "hashes": [ + "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9", + "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46" + ], + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==2.3.0" + }, + "filelock": { + "hashes": [ + "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81", + "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec" + ], + "markers": "python_version >= '3.7'", + "version": "==3.12.2" + }, + "freezegun": { + "hashes": [ + "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446", + "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f" + ], + "index": "pypi", + "version": "==1.2.2" + }, + "identify": { + "hashes": [ + "sha256:0aac67d5b4812498056d28a9a512a483f5085cc28640b02b258a59dac34301d4", + "sha256:986dbfb38b1140e763e413e6feb44cd731faf72d1909543178aa79b0e258265d" + ], + "markers": "python_version >= '3.7'", + "version": "==2.5.24" + }, + "iniconfig": { + "hashes": [ + "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", + "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.0" + }, + "nodeenv": { + "hashes": [ + "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2", + "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==1.8.0" + }, + "numpy": { + "hashes": [ + "sha256:012097b5b0d00a11070e8f2e261128c44157a8689f7dedcf35576e525893f4fe", + "sha256:0d3fe3dd0506a28493d82dc3cf254be8cd0d26f4008a417385cbf1ae95b54004", + "sha256:0def91f8af6ec4bb94c370e38c575855bf1d0be8a8fbfba42ef9c073faf2cf19", + "sha256:1a180429394f81c7933634ae49b37b472d343cccb5bb0c4a575ac8bbc433722f", + "sha256:1d5d3c68e443c90b38fdf8ef40e60e2538a27548b39b12b73132456847f4b631", + "sha256:20e1266411120a4f16fad8efa8e0454d21d00b8c7cee5b5ccad7565d95eb42dd", + "sha256:247d3ffdd7775bdf191f848be8d49100495114c82c2bd134e8d5d075fb386a1c", + "sha256:35a9527c977b924042170a0887de727cd84ff179e478481404c5dc66b4170009", + "sha256:38eb6548bb91c421261b4805dc44def9ca1a6eef6444ce35ad1669c0f1a3fc5d", + "sha256:3d7abcdd85aea3e6cdddb59af2350c7ab1ed764397f8eec97a038ad244d2d105", + "sha256:41a56b70e8139884eccb2f733c2f7378af06c82304959e174f8e7370af112e09", + "sha256:4a90725800caeaa160732d6b31f3f843ebd45d6b5f3eec9e8cc287e30f2805bf", + "sha256:6b82655dd8efeea69dbf85d00fca40013d7f503212bc5259056244961268b66e", + "sha256:6c6c9261d21e617c6dc5eacba35cb68ec36bb72adcff0dee63f8fbc899362588", + "sha256:77d339465dff3eb33c701430bcb9c325b60354698340229e1dff97745e6b3efa", + "sha256:791f409064d0a69dd20579345d852c59822c6aa087f23b07b1b4e28ff5880fcb", + "sha256:9a3a9f3a61480cc086117b426a8bd86869c213fc4072e606f01c4e4b66eb92bf", + "sha256:c1516db588987450b85595586605742879e50dcce923e8973f79529651545b57", + "sha256:c40571fe966393b212689aa17e32ed905924120737194b5d5c1b20b9ed0fb171", + "sha256:d412c1697c3853c6fc3cb9751b4915859c7afe6a277c2bf00acf287d56c4e625", + "sha256:d5154b1a25ec796b1aee12ac1b22f414f94752c5f94832f14d8d6c9ac40bcca6", + "sha256:d736b75c3f2cb96843a5c7f8d8ccc414768d34b0a75f466c05f3a739b406f10b", + "sha256:e8f6049c4878cb16960fbbfb22105e49d13d752d4d8371b55110941fb3b17800", + "sha256:f76aebc3358ade9eacf9bc2bb8ae589863a4f911611694103af05346637df1b7", + "sha256:fd67b306320dcadea700a8f79b9e671e607f8696e98ec255915c0c6d6b818503" + ], + "markers": "python_version >= '3.10'", + "version": "==1.25.1" + }, + "packaging": { + "hashes": [ + "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", + "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" + ], + "markers": "python_version >= '3.7'", + "version": "==23.1" + }, + "pandas": { + "hashes": [ + "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682", + "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc", + "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b", + "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089", + "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5", + "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26", + "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210", + "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b", + "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641", + "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd", + "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78", + "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b", + "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e", + "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061", + "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0", + "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e", + "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8", + "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d", + "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0", + "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c", + "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183", + "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df", + "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8", + "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f", + "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02" + ], + "index": "pypi", + "version": "==2.0.3" + }, + "platformdirs": { + "hashes": [ + "sha256:cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c", + "sha256:f87ca4fcff7d2b0f81c6a748a77973d7af0f4d526f98f308477c3c436c74d528" + ], + "markers": "python_version >= '3.7'", + "version": "==3.8.1" + }, + "pluggy": { + "hashes": [ + "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849", + "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3" + ], + "markers": "python_version >= '3.7'", + "version": "==1.2.0" + }, + "pre-commit": { + "hashes": [ + "sha256:0b4210aea813fe81144e87c5a291f09ea66f199f367fa1df41b55e1d26e1e2b4", + "sha256:5b808fcbda4afbccf6d6633a56663fed35b6c2bc08096fd3d47ce197ac351d9d" + ], + "index": "pypi", + "version": "==3.2.2" + }, + "pymongo": { + "hashes": [ + "sha256:01807a3d0b758cbf65e7b3af84a24d2ee58c9d6c0af8b0af99b581bcaa75a931", + "sha256:02060ced24a26e1c73b6f491b728fe99e73f38ba3a1e4b882dc7b873d419ab3e", + "sha256:028addb304bc525d4a10c5c6e59ef5f140e528ae285c10e1d43f19905631e32f", + "sha256:02f535bc8f8d75d45ec6cd944804d466a73a46afc368d6c36e232b887edd0475", + "sha256:0669823de06c3a77fddf738f6250688b7fdae2b44edbe3c103b7fbfdfc848392", + "sha256:071c256fbb35c6942970b8b6eb6b89bac302db49a2d6d35e68c35b442a0ce710", + "sha256:093c5343c5863e87023318050507511fa7458b0376caabcc41abff0e36aaabc8", + "sha256:15cf004b6329da48078d7d9d1c79c802df6631b94e5a1ed9a112d713cc0f66e9", + "sha256:18acb807de39eb9b8ff7122094920f1da79c1781dc96cfef73dd97da51448f7b", + "sha256:196c2c9ffccdf0ad4efdfae29347c4e2ae52c3415e958736cda84e4062553e96", + "sha256:1a1bb096579ffa59143a8d8fc9d4692db3e04305cf5a0e48e0724ae47a836255", + "sha256:1c6bd8470c89b2cd6312fa685dbf4c64371a04a7e4a3a55e2007626f8f997103", + "sha256:1eea8af506b8b33573c76942a5c2390f2cddb4e195b1cdfc373ca919e9b95904", + "sha256:2128592140426044d59a89f30b7aba1e185faf2553b3d161dcca0aa970ba40c7", + "sha256:23bfd793be088470a1c7bca5c907ae3180e6a4cf731e96a194c89413c042cf4c", + "sha256:242d1a30162ead28e69df37748021039c4b292bbfd7c5449294a26c8365d342d", + "sha256:2578f077b9448b7a420b3e9b0efdfb7ecdb2a3c27e00c181610809717c900cd9", + "sha256:274eb75231aca12d54d421852fc230e8655e4b33b30e9eb7fd34269955e125dd", + "sha256:29956f61ab885c28b190ff817de7ad0c75a470699632b44848b102640fbf9e73", + "sha256:2caac57d2a0160dce877e706e94e8a15b87feb71c257ecb8b5a039f7e98ba99b", + "sha256:2f74b606c11c570ec2a6c384fc194d96f00eaa829c7c08cbec455f7b02d28774", + "sha256:34ea6ffb77f0cf8d01c4c1df60dc68141859ada1507c326380ef81e23b58c9cc", + "sha256:38ece8d2892de19fa437bc4f60b0d8c5353b185e8cc1c543212a488c93c74834", + "sha256:3e6efcf768209dc4d966fabbbe4dcd2dd2d60ec4d8342668da02664f0c73a9e8", + "sha256:40ad38ad6f6dbd8a9dbd05195a15fe7749a0692dc895274a4f806d9831fceb3c", + "sha256:4481f2796d53cd0c74d988a23a11266e6cae03be3878f42ed2c221b192d14f8d", + "sha256:44893b6788da1d67696ff2f27e42e315d40965a4fa23786dcc26c932c5b02149", + "sha256:45838fa9abf3bce35a24fffcd289de23f3d6edc4cc9beac28037df3e1009f892", + "sha256:48908eaca3dccc2af8b4eae73ee00d2e1e7ffe91ce630c8906981c075161ad8c", + "sha256:4a0cfab6b6c1826e8dfe4453c08aa70343a693dede7c09dca564a9b1f2393374", + "sha256:4a28ad09abccc9f71632398febfea12d3f28cec7e44fe6f2b16665807e62c298", + "sha256:4b092e2a11f37a41e1767a221ff2719397ae2e033f978de236ce10c4b1916227", + "sha256:4b43ae6e1c4b972761065f77f3eff4b914154bc5bd74d632305875c5309eafd1", + "sha256:4b65f4e66efe43dcc5fb3a285f899e798742b8365bafdd832054675d34d640d5", + "sha256:4eba5abcee347bdaa7f1a3f18fd97758f0b75a6dc5704885e793aeb51e8e5e32", + "sha256:50294bae0f20ec4f8d3f5eefd133956f582942c156d08f6b88f2a1b1efe04c53", + "sha256:5e13ba36f18489600db28da13da73e8e190bd48899ad268cb482fe726d31a922", + "sha256:67aa85bbab597615efd83f521c8da34dd9a19b7456cc919c227378c793073183", + "sha256:6a2564ed1a07258a73f7adfb0663aa69022f1edc431d11aae4a32a29e7755d3c", + "sha256:6aa18b255af46641d167378f8b8f06becb6eb1670f622aefa34e502362267fa9", + "sha256:6acedf0778b79b6ea111a28fb23760b5f6b7b1c3e1f1e3595cf87ce709bce344", + "sha256:78be52dc21f578a17e2c1cf1a222d4e64e91e0b1dba7e18f5ff7be7c0bf8053f", + "sha256:7e0fbf05bb74a3f610f970a178bfb4e048f6b82fc22dda5e14e0ddfc4d66c9b7", + "sha256:7eb221dcb9e27415d2bd6e2d3001d1da0f351e2aa1564f6f3987f2206c066600", + "sha256:801094c80d117b0d476f0afbe16cdfe438cc4069c5b0f2578564cb4b3c83f80f", + "sha256:88ceab5cd84f7d86f018fa66377d6f90fcf3643d56283f2f4124ccef58390a0e", + "sha256:8b6603315f9a15e5ed80143a5a744c695a8503e27a76fb5828f7269225f39ddd", + "sha256:8d8a8aef8724058d416536902e680f2b06499e58c54220becdfcd3ff8e5dccfd", + "sha256:8edb59aa5c10a3fb8d3a0c2cac8ba58c0d8f4e56f9003378ac1fff503a8d3f42", + "sha256:8fd68b540fb70954deeb2b6a1fb2f34d6342bcf221e003e6063e3b28e87b2778", + "sha256:900c773d4f9d68747bb19ef40c35c701f4a919a6b96efa2d4e1cb50074a9738e", + "sha256:93d8d5ee37cd9634747a1363496fd3949451bdaeb9539278985cb6fd08d929cf", + "sha256:95a5036b7046f617207f38936c188eeb56dbe740cba0fa1215df2e1b9ee38c74", + "sha256:9f3e8fc3433a86ab0b3d960f8fb48fe7135876df04987dd04b3bf35d9b49ae92", + "sha256:a1b5d286fee4b9b5a0312faede02f2ce2f56ac695685af1d25f428abdac9a22c", + "sha256:a2875f0bdb605e56630f46e12082f26ac2c680a5473f5f154b7131841727948c", + "sha256:a2e496753e91bc82dfbe1f3bab21e0907866dab3c810e255ebaf991cd5c5455d", + "sha256:a4315509a4e155e6bd4b199bd435ff2bb31f558915726e0c50a725ae7b99727f", + "sha256:a5551491ace0f05ae0bbe5a496c4daf216d9fc98e182940f471c228235b1626e", + "sha256:a8d482e2ae01a5ac17183afe8c808cb6919889bdf22f0d3663105ccf0ea89adf", + "sha256:b0ddb34591f5e19296ef5b643e23ec5179a7c1d2b73c17701f50dcfa493e0252", + "sha256:b100895d6b57d6a7e8de5fd15578aaa46170b56d978baf56182c10e8ba725fbf", + "sha256:b213fae58d6ba14ac71a481691981e582ff5813630f3a82aaf92fb79399ba0ec", + "sha256:b3eed06a24157a891eac5f73ec2400d22cccc95cde78a3f0e2b90c5ab17f1cf1", + "sha256:b641683de9bf05b4b52a629bf8ddd5fa0fb061ca54bc2412ce89ce2de2beda36", + "sha256:bf4e83af0bd3cf4c98eaf1ed2d028afd520bdffd6f7570f6cc5a44e9363fbb9a", + "sha256:c96a080cae86c1c27758fdd3fbee0298a006b05272de4dff9dea21ca34952c72", + "sha256:cc5c56169effa5bf9fae5e9a66efc211b3f252869d99d6c400792eced7f213b9", + "sha256:d0a8f16a97758ca9af1baa927521b24175dba7e95ce745d5bf64a5c75fe61df8", + "sha256:d368def4564681f681f4fe1ae906239bb4dc7dd403c49d15d3a6fe2688950f13", + "sha256:e5f19bb887d83959ba1c359fba16cdedb0f868ba85ae375c3e4e0fdf5697a524", + "sha256:ebe1954aa85e622674ea01828419f129527c95c40a392e0f7761e242d85a772f", + "sha256:f38bbab4d13d180ed00c2f107e36503bd2e2cc4c6d4ae2734c0a85c2edaf2d2e", + "sha256:ff281a66925790a05e3c7e0de1350a0992b66a4e51724317ac35026ac856ae28" + ], + "index": "pypi", + "version": "==4.4.0" + }, + "pytest": { + "hashes": [ + "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32", + "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a" + ], + "index": "pypi", + "version": "==7.4.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", + "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" + ], + "index": "pypi", + "version": "==2023.3" + }, + "pyyaml": { + "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0" + }, + "setuptools": { + "hashes": [ + "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", + "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" + ], + "markers": "python_version >= '3.7'", + "version": "==68.0.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:00aa050faf24ce5f2af643e2b86822fa1d7149649995f11bc1e769bbfbf9010b", + "sha256:09397a18733fa2a4c7680b746094f980060666ee549deafdb5e102a99ce4619b", + "sha256:0f7fdcce52cd882b559a57b484efc92e108efeeee89fab6b623aba1ac68aad2e", + "sha256:10514adc41fc8f5922728fbac13d401a1aefcf037f009e64ca3b92464e33bf0e", + "sha256:10e001a84f820fea2640e4500e12322b03afc31d8f4f6b813b44813b2a7c7e0d", + "sha256:194f2d5a7cb3739875c4d25b3fe288ab0b3dc33f7c857ba2845830c8c51170a0", + "sha256:1aac42a21a7fa6c9665392c840b295962992ddf40aecf0a88073bc5c76728117", + "sha256:1fb792051db66e09c200e7bc3bda3b1eb18a5b8eb153d2cedb2b14b56a68b8cb", + "sha256:2756485f49e7df5c2208bdc64263d19d23eba70666f14ad12d6d8278a2fff65f", + "sha256:2b791577c546b6bbd7b43953565fcb0a2fec63643ad605353dd48afbc3c48317", + "sha256:420bc6d06d4ae7fb6921524334689eebcbea7bf2005efef070a8562cc9527a37", + "sha256:45b07470571bda5ee7f5ec471271bbde97267cc8403fce05e280c36ea73f4754", + "sha256:4ebc542d2289c0b016d6945fd07a7e2e23f4abc41e731ac8ad18a9e0c2fd0ec2", + "sha256:556dc18e39b6edb76239acfd1c010e37395a54c7fde8c57481c15819a3ffb13e", + "sha256:589aba9a35869695b319ed76c6f673d896cd01a7ff78054be1596df7ad9b096f", + "sha256:5c95e3e7cc6285bf7ff263eabb0d3bfe3def9a1ff98124083d45e5ece72f4579", + "sha256:5dd574a37be388512c72fe0d7318cb8e31743a9b2699847a025e0c08c5bf579d", + "sha256:67fbb40db3985c0cfb942fe8853ad94a5e9702d2987dec03abadc2f3b6a24afb", + "sha256:6852cd34d96835e4c9091c1e6087325efb5b607b75fd9f7075616197d1c4688a", + "sha256:69ae0e9509c43474e33152abe1385b8954922544616426bf793481e1a37e094f", + "sha256:6c5bae4c288bda92a7550fe8de9e068c0a7cd56b1c5d888aae5b40f0e13b40bd", + "sha256:774bd401e7993452ba0596e741c0c4d6d22f882dd2a798993859181dbffadc62", + "sha256:79228a7b90d95957354f37b9d46f2cc8926262ae17b0d3ed8f36c892f2a37e06", + "sha256:7b8cba5a25e95041e3413d91f9e50616bcfaec95afa038ce7dc02efefe576745", + "sha256:7db97eabd440327c35b751d5ebf78a107f505586485159bcc87660da8bb1fdca", + "sha256:7ddd6d35c598af872f9a0a5bce7f7c4a1841684a72dab3302e3df7f17d1b5249", + "sha256:82edf3a6090554a83942cec79151d6b5eb96e63d143e80e4cf6671e5d772f6be", + "sha256:8b7b3ebfa9416c8eafaffa65216e229480c495e305a06ba176dcac32710744e6", + "sha256:8da677135eff43502b7afab5a1e641edfb2dc734ba7fc146e9b1b86817a728e2", + "sha256:908c850b98cac1e203ababd4ba76868d19ae0d7172cdc75d3f1b7829b16837d2", + "sha256:9da4ee8f711e077633730955c8f3cd2485c9abf5ea0f80aac23221a3224b9a8c", + "sha256:a6f1d8256d06f58e6ece150fbe05c63c7f9510df99ee8ac37423f5476a2cebb4", + "sha256:afb322ca05e2603deedbcd2e9910f11a3fd2f42bdeafe63018e5641945c7491c", + "sha256:b52c6741073de5a744d27329f9803938dcad5c9fee7e61690c705f72973f4175", + "sha256:ba633b51835036ff0f402c21f3ff567c565a22ff0a5732b060a68f4660e2a38f", + "sha256:bfa1a0f83bdf8061db8d17c2029454722043f1e4dd1b3d3d3120d1b54e75825a", + "sha256:bffd6cd47c2e68970039c0d3e355c9ed761d3ca727b204e63cd294cad0e3df90", + "sha256:d7a2c1e711ce59ac9d0bba780318bcd102d2958bb423209f24c6354d8c4da930", + "sha256:da46beef0ce882546d92b7b2e8deb9e04dbb8fec72945a8eb28b347ca46bc15a", + "sha256:ebdd2418ab4e2e26d572d9a1c03877f8514a9b7436729525aa571862507b3fea", + "sha256:fc44e50f9d5e96af1a561faa36863f9191f27364a4df3eb70bca66e9370480b6" + ], + "index": "pypi", + "version": "==2.0.18" + }, + "typing-extensions": { + "hashes": [ + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" + ], + "markers": "python_version >= '3.7'", + "version": "==4.7.1" + }, + "tzdata": { + "hashes": [ + "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a", + "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda" + ], + "markers": "python_version >= '2'", + "version": "==2023.3" + }, + "virtualenv": { + "hashes": [ + "sha256:34da10f14fea9be20e0fd7f04aba9732f84e593dac291b757ce42e3368a39419", + "sha256:8ff19a38c1021c742148edc4f81cb43d7f8c6816d2ede2ab72af5b84c749ade1" + ], + "markers": "python_version >= '3.7'", + "version": "==20.23.1" + } + }, + "develop": {} +} diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dao/__init__.py b/dao/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dao/_base_mongo.py b/dao/_base_mongo.py new file mode 100644 index 0000000..364f268 --- /dev/null +++ b/dao/_base_mongo.py @@ -0,0 +1,26 @@ +from pymongo import MongoClient + +from sample_code.settings import ( + ARC_MONGO_AUTHMECHANISM, + ARC_MONGO_AUTHSOURCE, + ARC_MONGO_READ_PREFERENCE, +) + + +class BaseMongoDAO: + def __init__( + self, + mongoServers: str, + mongoReplicaset: str, + username: str, + password: str, + database: str, + ) -> None: + mongo_uri = f"mongodb://{username}:{password}@{mongoServers}" + self.client = MongoClient( + mongo_uri, + replicaSet=mongoReplicaset, + authSource=ARC_MONGO_AUTHSOURCE, + readPreference=ARC_MONGO_READ_PREFERENCE, + authMechanism=ARC_MONGO_AUTHMECHANISM, + )[database] diff --git a/dao/audit.py b/dao/audit.py new file mode 100644 index 0000000..a82d80b --- /dev/null +++ b/dao/audit.py @@ -0,0 +1,102 @@ +import logging +from datetime import datetime + +from pymongo.collection import Collection +from pymongo.errors import PyMongoError + +from sample_code.dao._base_mongo import BaseMongoDAO +from sample_code.settings import ( + AUDIT_COLLECTION, + AUDIT_DATABASE, + AUDIT_PASSWORD, + AUDIT_USERNAME, +) + +logger = logging.getLogger(__name__) + + +class AuditDAO(BaseMongoDAO): + def __init__( + self, + mongoServers: str, + mongoReplicaset: str, + ) -> None: + super().__init__( + username=AUDIT_USERNAME, + password=AUDIT_PASSWORD, + database=AUDIT_DATABASE, + mongoServers=mongoServers, + mongoReplicaset=mongoReplicaset, + ) + + def run_aggregation_query(collection: Collection, query: str, **kwargs): + try: + return collection.aggregate(query, **kwargs) + except PyMongoError as exc: + logger.error(str(exc)) + + def get_subscribers(self, auditRangeStart: datetime, auditRangeEnd: datetime): + logger.info( + f"Get subscriber usage between {auditRangeStart.isoformat()} and {auditRangeEnd.isoformat()}" + ) + auditCollection = self.client[AUDIT_COLLECTION] + auditQuery = [ + { + "$match": { + "$and": [ + { + "details": { + "$elemMatch": { + "state": "ADD", + "data.payload.payloads": { + "$elemMatch": { + "requestpayload.subscriptions": { + "$elemMatch": { + "offerName": "MYOFFERNAME" + } + } + } + }, + } + } + }, + { + "lastModifiedDate": { + "$gte": auditRangeStart, + "$lte": auditRangeEnd, + } + }, + ] + } + }, + {"$unwind": {"path": "$details"}}, + { + "$match": { + "details.state": "ADD", + "details.data.payload.payloads": { + "$elemMatch": { + "requestpayload.subscriptions": { + "$elemMatch": {"offerName": "MYOFFERNAME"} + } + } + }, + } + }, + {"$unwind": {"path": "$details.data.payload.payloads"}}, + { + "$unwind": { + "path": "$details.data.payload.payloads.requestpayload.subscriptions" + } + }, + { + "$project": { + "_id": 0.0, + "ban": 1.0, + "subscriberId": "$details.data.payload.subscriberId", + "effectiveDate": "$details.data.payload.payloads.requestpayload.subscriptions.effectiveDate", + "expiryDate": "$details.data.payload.payloads.requestpayload.subscriptions.expiryDate", + } + }, + ] + + return self.run_aggregation_query(auditCollection, auditQuery, cursor={}) diff --git a/dao/reporting.py b/dao/reporting.py new file mode 100644 index 0000000..472769a --- /dev/null +++ b/dao/reporting.py @@ -0,0 +1,62 @@ +import logging + +from sqlalchemy import create_engine +from sqlalchemy.exc import SQLAlchemyError + +from sample_code.settings import ( + REPORTING_AULDATALEAK_TABLENAME, + REPORTING_SQL_DATABASE, + REPORTING_SQL_PASSWORD, + REPORTING_SQL_PORT, + REPORTING_SQL_SERVER, + REPORTING_SQL_USERNAME, +) + +logger = logging.getLogger(__name__) + + +class ReportDAO: + def __init__(self) -> None: + mysql_uri = f"mysql://{REPORTING_SQL_USERNAME}:{REPORTING_SQL_PASSWORD}@{REPORTING_SQL_SERVER}:{REPORTING_SQL_PORT}/{REPORTING_SQL_DATABASE}?charset=utf8" + self.client = create_engine(mysql_uri, pool_recycle=3600) + + def run_query(self, query): + try: + self.client.execute(query) + return 0 + except SQLAlchemyError as e: + error = str(e.__dict__["orig"]) + logger.error(error) + + def create_reporting_table(self) -> None: + logger.info("Initiate reporting table") + reportingTableCreateQuery = f"CREATE TABLE IF NOT EXISTS {REPORTING_AULDATALEAK_TABLENAME} ( \ + `SUBSCRIBERID` VARCHAR(100), \ + `MDN` VARCHAR(100), \ + `BAN` VARCHAR(100), \ + `USAGESTART` DATETIME, \ + `USAGEEND` DATETIME, \ + `TOTALMB` DECIMAL, \ + `AUDITDATE` DATETIME \ + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;" + + reportingTableCreateIndex = f"CREATE INDEX idx_AUDITDATE \ + ON {REPORTING_AULDATALEAK_TABLENAME} (AUDITDATE);" + + self.run_query(reportingTableCreateQuery) + self.run_query(reportingTableCreateIndex) + + @staticmethod + def process_data_for_insert(rows: list) -> str: + return ", ".join([f"({', '.join(map(str, r))})" for r in rows]) + + def insert_reporting_data(self, rows: list) -> None: + logger.info("Insert new data in reporting table") + usageReportingQuery = f"INSERT INTO {REPORTING_AULDATALEAK_TABLENAME} (SUBSCRIBERID, MDN, BAN, USAGESTART, USAGEEND, TOTALMB, AUDITDATE) VALUES " + data = self.process_data_for_insert(rows) + self.run_query(usageReportingQuery + data) + + def clean_reporting_data(self) -> None: + logger.info("Clean reporting table") + reportingTableDeleteQuery = f"DELETE FROM {REPORTING_AULDATALEAK_TABLENAME} WHERE AUDITDATE < DATE_SUB(NOW(), INTERVAL 1 MONTH)" + self.run_query(reportingTableDeleteQuery) diff --git a/dao/usage.py b/dao/usage.py new file mode 100644 index 0000000..dff2bd4 --- /dev/null +++ b/dao/usage.py @@ -0,0 +1,78 @@ +import logging +from datetime import datetime + +from pymongo import DESCENDING +from pymongo.collection import Collection +from pymongo.errors import PyMongoError + +from sample_code.dao._base_mongo import BaseMongoDAO +from sample_code.settings import COLLECTION, DATABASE, PASSWORD, USERNAME + +logger = logging.getLogger(__name__) + + +class UsageDAO(BaseMongoDAO): + def __init__( + self, + mongoServers: str, + mongoReplicaset: str, + ) -> None: + super().__init__( + username=USERNAME, + password=PASSWORD, + database=DATABASE, + mongoServers=mongoServers, + mongoReplicaset=mongoReplicaset, + ) + + def run_query( + collection: Collection, + query: dict, + project: dict = None, + sort: bool = True, + sort_field: str = "eventTime", + limit_results: bool = False, + limit_count: int = 10, + ) -> list: + try: + if project is not None: + db_query = collection.find(query, project) + else: + db_query = collection.find(query) + + if sort: + db_query.sort(sort_field, DESCENDING) + + if limit_results: + db_query.limit(limit_count) + + return [doc for doc in db_query] + except PyMongoError as exc: + logger.error(str(exc)) + + def get_subscriber_usage( + self, subscriberId: str, effectiveDate: datetime, expiryDate: datetime + ) -> list: + logger.info( + f"Get subscriber usage between {effectiveDate.isoformat()} and {expiryDate.isoformat()}" + ) + collection = self.client[COLLECTION] + usageQuery = { + "$and": [ + {"end": {"$gte": effectiveDate, "$lte": expiryDate}}, + {"extSubId": eval(subscriberId)}, + {"usageType": "OVER"}, + {"$or": [{"bytesIn": {"$gt": 0}, "bytesOut": {"$gt": 0}}]}, + ] + } + usageProject = { + "_id": 0, + "extSubId": 1, + "MDN": 1, + "BAN": 1, + "start": 1, + "end": 1, + "bytesIn": 1, + "bytesOut": 1, + } + return self.run_query(collection, usageQuery, usageProject) diff --git a/main.py b/main.py new file mode 100644 index 0000000..599c2cc --- /dev/null +++ b/main.py @@ -0,0 +1,142 @@ +import logging +from datetime import date, datetime, time, timedelta + +import pandas as pd +import pytz + +from sample_code.dao.audit import AuditDAO +from sample_code.dao.reporting import ReportDAO +from sample_code.dao.usage import UsageDAO +from sample_code.settings import ( + AUDIT_REPLICASET, + AUDIT_SERVER, + REPLICASET_A, + REPLICASET_B, + REPLICASET_C, + SERVER_A, + SERVER_B, + SERVER_C, +) + +logger = logging.getLogger(__name__) + + +class Main: + def __init__(self) -> None: + logger.info("Initiate clients with databases") + self.reportingClient = ReportDAO() + + self.auditClient = AuditDAO( + mongoServers=AUDIT_SERVER, + mongoReplicaset=AUDIT_REPLICASET, + ) + + self.usageClient_A = UsageDAO( + mongoServers=SERVER_A, + mongoReplicaset=REPLICASET_A, + ) + + self.usageClient_B = UsageDAO( + mongoServers=SERVER_B, + mongoReplicaset=REPLICASET_B, + ) + + self.usageClient_C = UsageDAO( + mongoServers=SERVER_C, + mongoReplicaset=REPLICASET_C, + ) + + def get_auldata_subscribers( + self, auditRangeStart: datetime, auditRangeEnd: datetime + ): + logger.info( + f"Get subscribers for the range between {auditRangeStart.isoformat()} and {auditRangeEnd.isoformat()}" + ) + res = self.auditClient.get_subscribers(auditRangeStart, auditRangeEnd) + return pd.DataFrame(list(res)) + + def compare(self, auldataSubs): + logger.info(f"Start comparing subscribers's data") + subListA = [] + subListB = [] + subListC = [] + + for _, row in auldataSubs.iterrows(): + remainder = int(row["ban"]) % 3 + if remainder == 0: + subListA.append(row) + elif remainder == 1: + subListB.append(row) + elif remainder == 2: + subListC.append(row) + + self.run_compare_on_node("A", subListA) + self.run_compare_on_node("B", subListB) + self.run_compare_on_node("C", subListC) + + def run_compare_on_node(self, node: str, subList: list): + logger.info(f"Start comparing subscribers's data on the node {node}") + to_date = lambda d: datetime.strptime(d, "%Y-%m-%dT%H:%M:%SZ").astimezone( + pytz.timezone("US/Eastern") + ) + + if len(subList) > 0: + auditDate = datetime.today().strftime("%Y-%m-%d %H:%M:%S") + usageClient = getattr(self, f"usageClient_{node}", None) + + if not usageClient: + raise Exception("Wrong node!") + + usageResult = pd.DataFrame( + columns=[ + "extSubId", + "MDN", + "BAN", + "start", + "end", + "bytesIn", + "bytesOut", + ] + ) + + for subscriber in subList: + effectiveDate = to_date(subscriber["effectiveDate"]) + expiryDate = to_date(subscriber["expiryDate"]) + + res = usageClient.get_subscriber_usage( + subscriber["subscriberId"], effectiveDate, expiryDate + ) + usageResult = pd.concat([usageResult, pd.DataFrame(res)], axis=0) + + if len(usageResult) > 0: + data = [ + [ + row["extSubId"], + row["MDN"], + row["BAN"], + row["start"], + row["end"], + int(row["bytesIn"]) + int(row["bytesOut"]), + auditDate, + ] + for _, row in usageResult.iterrows() + ] + + self.reportingClient.insert_reporting_data(data) + + +if __name__ == "__main__": + logger.info("Start the main script") + + mainClient = Main() + mainClient.reportingClient.create_reporting_table() + + auditDate = date.today() - timedelta(days=1) + auditRangeStart = datetime.combine(auditDate, time(0, 0, 0)) + auditRangeEnd = datetime.combine(auditDate, time(23, 59, 59)) + + auldataSubs = mainClient.get_auldata_subscribers(auditRangeStart, auditRangeEnd) + mainClient.compare(auldataSubs) + + mainClient.reportingClient.clean_reporting_data() + logger.info("Finish the main script") diff --git a/sample_script.py b/sample_script.py deleted file mode 100644 index 7454a6d..0000000 --- a/sample_script.py +++ /dev/null @@ -1,269 +0,0 @@ -import os - -import pytz -import pandas as pd -from datetime import datetime, timedelta, date, time -from pymongo import MongoClient -from pymongo.collection import Collection -from pymongo import DESCENDING -from sqlalchemy import create_engine -from sqlalchemy.exc import SQLAlchemyError - -REPORTING_SQL_SERVER = '127.0.0.1' -REPORTING_SQL_PORT = '3306' -REPORTING_SQL_DATABASE = 'myreportingdatabase' -REPORTING_SQL_USERNAME = os.environ.get('REPORTING_SQL_USERNAME') -REPORTING_SQL_PASSWORD = os.environ.get('REPORTING_SQL_PASSWORD') - -AUDIT_SERVER = "127.0.0.1:27018" -AUDIT_REPLICASET = "rs4" -AUDIT_USERNAME = os.environ.get('MONGO_AUDIT_USERNAME') -AUDIT_PASSWORD = os.environ.get('MONGO_AUDIT_PASSWORD') -AUDIT_DATABASE = "mydb" -AUDIT_COLLECTION = "myauditcollection" - -SERVER_A = "127.0.0.1:27017" -SERVER_B = "127.0.0.1:27017" -SERVER_C = "127.0.0.1:27017" -REPLICASET_A = "rs0" -REPLICASET_B = "rs1" -REPLICASET_C = "rs2" -USERNAME = os.environ.get('mongo_USERNAME')PASSWORD = os.environ.get('mongo_PASSWORD')DATABASE = "mydb"COLLECTION = "mycollection" -ARC_MONGO_PORT = '27017' -ARC_MONGO_AUTHMECHANISM = "SCRAM-SHA-1" -ARC_MONGO_AUTHSOURCE = "admin" -ARC_MONGO_DATABASE = 'admin' -ARC_MONGO_READ_PREFERENCE = "secondary" - -REPORTING_AULDATALEAK_TABLENAME = "auldata_leak" - - -def get_mongo_client(mongoServers: str, mongoReplicaset: str, username: str, password: str): - mongo_uri = 'mongodb://%s:%s@%s' % (username, password, mongoServers) - return MongoClient(mongo_uri, replicaSet=mongoReplicaset, authSource=ARC_MONGO_AUTHSOURCE, - readPreference=ARC_MONGO_READ_PREFERENCE, - authMechanism=ARC_MONGO_AUTHMECHANISM) - - -def connect_to_mysql(): - mysql_uri = 'mysql://%s:%s@%s:%s/%s?charset=utf8' % (REPORTING_SQL_USERNAME, REPORTING_SQL_PASSWORD, - REPORTING_SQL_SERVER, REPORTING_SQL_PORT, - REPORTING_SQL_DATABASE) - return create_engine(mysql_uri, pool_recycle=3600) - - -def run_mongo_query(collection: Collection, query: dict, project: dict = None, sort: bool = True, - sort_field: str = 'eventTime', - limit_results: bool = False, limit_count: int = 10): - results = [] - if project is not None: - db_query = collection.find(query, project) - else: - db_query = collection.find(query) - if sort: - db_query.sort(sort_field, DESCENDING) - if limit_results: - db_query.limit(limit_count) - for doc in db_query: - results.append(doc) - - results_df = pd.DataFrame(list(results)) - return results_df - - -def run_mongo_query_agr(collection: Collection, query: dict): - results = collection.aggregate(query, cursor={}) - results_df = pd.DataFrame(list(results)) - return results_df - - -def create_mysql_table(sql_client, q, tbl_name): - try: - sql_client.execute(q) - return 0 - except SQLAlchemyError as e: - error = str(e.__dict__['orig']) - return error - - -def init_aludata_leak_reporting_table(client): - print('Creating table... ' + REPORTING_AULDATALEAK_TABLENAME) - - reportingTableCreateQuery = f'CREATE TABLE IF NOT EXISTS {REPORTING_AULDATALEAK_TABLENAME} ( \ - `SUBSCRIBERID` VARCHAR(100), \ - `MDN` VARCHAR(100), \ - `BAN` VARCHAR(100), \ - `USAGESTART` DATETIME, \ - `USAGEEND` DATETIME, \ - `TOTALMB` DECIMAL, \ - `AUDITDATE` DATETIME \ - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;' - - reportingTableCreateIndex = f'CREATE INDEX idx_AUDITDATE \ - ON {REPORTING_AULDATALEAK_TABLENAME} (AUDITDATE);' - - create_mysql_table(client, reportingTableCreateQuery, REPORTING_AULDATALEAK_TABLENAME) - create_mysql_table(client, reportingTableCreateIndex, REPORTING_AULDATALEAK_TABLENAME) - - -def get_auldata_subscribers(auditRangeStart: datetime, auditRangeEnd: datetime): - auditClient = get_mongo_client( - mongoServers=AUDIT_SERVER, - mongoReplicaset=AUDIT_REPLICASET, - username=AUDIT_USERNAME, - password=AUDIT_PASSWORD)[ARC_AUDIT_DATABASE] - auditCollection = auditClient[AUDIT_COLLECTION] - - # print(auditRangeStart.strftime('%Y-%m-%dT%H:%M:%SZ')) - # print(auditRangeEnd.strftime('%Y-%m-%dT%H:%M:%SZ')) - - auditQuery = [ - { - "$match": { - "$and": [ - { - "details": { - "$elemMatch": { - "state": "ADD", - "data.payload.payloads": { - "$elemMatch": { - "requestpayload.subscriptions": { - "$elemMatch": { - "offerName": "MYOFFERNAME" - } - } - } - } - } - } - }, - { - "lastModifiedDate": { - "$gte": auditRangeStart, - "$lte": auditRangeEnd - } - } - ] - } - }, - { - "$unwind": { - "path": "$details" - } - }, - { - "$match": { - "details.state": "ADD", - "details.data.payload.payloads": { - "$elemMatch": { - "requestpayload.subscriptions": { - "$elemMatch": { - "offerName": "MYOFFERNAME" - } - } - } - } - } - }, - { - "$unwind": { - "path": "$details.data.payload.payloads" - } - }, - { - "$unwind": { - "path": "$details.data.payload.payloads.requestpayload.subscriptions" - } - }, - { - "$project": { - "_id": 0.0, - "ban": 1.0, - "subscriberId": "$details.data.payload.subscriberId", - "effectiveDate": "$details.data.payload.payloads.requestpayload.subscriptions.effectiveDate", - "expiryDate": "$details.data.payload.payloads.requestpayload.subscriptions.expiryDate" - } - } - ] - - return run_mongo_query_agr(auditCollection, auditQuery) - - -def run_compare_on_node(node: str, subList): - auditDate = datetime.today().strftime('%Y-%m-%d %H:%M:%S') - arcUsageServer = "" - arcUsageReplicaset = "" - - if node == "A": - arcUsageServer = SERVER_A - arcUsageReplicaset = REPLICASET_A - elif node == "B": - arcUsageServer = SERVER_B - arcUsageReplicaset = REPLICASET_B - elif node == "C": - arcUsageServer = SERVER_C - arcUsageReplicaset = REPLICASET_C - - if len(subList) > 0: - usageClient = get_mongo_client( - mongoServers=arcUsageServer, - mongoReplicaset=arcUsageReplicaset, - username=USERNAME, password=PASSWORD)[ARC_USAGE_DATABASE] usageCollection = usageClient[COLLECTION] - usageResult = pd.DataFrame(columns = ['extSubId', 'MDN', 'BAN', 'start', 'end', 'bytesIn', 'bytesOut']) - - for subscriber in subList: - effectiveDate = datetime.strptime(subscriber["effectiveDate"], '%Y-%m-%dT%H:%M:%SZ').astimezone(pytz.timezone('US/Eastern')) - expiryDate = datetime.strptime(subscriber["expiryDate"], '%Y-%m-%dT%H:%M:%SZ').astimezone(pytz.timezone('US/Eastern')) - - usageQuery = {"$and": [ - {"end": {"$gte": effectiveDate, "$lte": expiryDate}}, - {"extSubId": eval(subscriber["subscriberId"])}, - {"usageType": "OVER"}, - {"$or": [{"bytesIn": {"$gt": 0}, "bytesOut": {"$gt": 0}}]} - ]} - usageProject = {"_id": 0, "extSubId": 1, "MDN": 1, "BAN": 1, "start": 1, "end": 1, "bytesIn": 1, "bytesOut": 1} - queryResult = run_mongo_query(usageCollection, usageQuery, usageProject) - usageResult = pd.concat([usageResult, queryResult], axis=0) - - if len(usageResult) > 0: - usageResultReportingQuery = f"INSERT INTO {REPORTING_AULDATALEAK_TABLENAME} (SUBSCRIBERID, MDN, BAN, USAGESTART, USAGEEND, TOTALMB, AUDITDATE) VALUES " - for index, row in usageResult.iterrows(): - usageResultReportingQuery = usageResultReportingQuery + f"(\'{row['extSubId']}\', {row['MDN']}, {row['BAN']}, \'{row['start']}\', \'{row['end']}\', \'{int(row['bytesIn']) + int(row['bytesOut'])}\', \'{auditDate}\')," - usageResultReportingQuery = usageResultReportingQuery[:-1] - reportingClient.execute(usageResultReportingQuery) - print(usageResult.size + " rows written to " + REPORTING_AULDATALEAK_TABLENAME) - -def compare(auldataSubs): - subListA = [] - subListB = [] - subListC = [] - - for index, row in auldataSubs.iterrows(): - remainder = int(row["ban"]) % 3 - if remainder == 0: - subListA.append(row) - elif remainder == 1: - subListB.append(row) - elif remainder == 2: - subListC.append(row) - - run_compare_on_node("A", subListA) - run_compare_on_node("B", subListB) - run_compare_on_node("C", subListC) - - -def aludata_leak_reporting_table_cleanup(client): - reportingTableDeleteQuery = f"DELETE FROM {REPORTING_AULDATALEAK_TABLENAME} WHERE AUDITDATE < DATE_SUB(NOW(), INTERVAL 1 MONTH)" - client.execute(reportingTableDeleteQuery) - - -if __name__ == '__main__': - reportingClient = connect_to_mysql() - init_aludata_leak_reporting_table(reportingClient) - auditDate = date.today() - timedelta(days=1) - auditRangeStart = (datetime.combine(auditDate, time(0, 0, 0))) - auditRangeEnd = (datetime.combine(auditDate, time(23, 59, 59))) - - auldataSubs = get_auldata_subscribers(auditRangeStart, auditRangeEnd) - compare(auldataSubs) - aludata_leak_reporting_table_cleanup(reportingClient) diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..0bff27d --- /dev/null +++ b/settings.py @@ -0,0 +1,42 @@ +import logging +import os + +logging.basicConfig( + level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) + +REPORTING_SQL_SERVER = os.environ.get("REPORTING_SQL_SERVER", "127.0.0.1") +REPORTING_SQL_PORT = os.environ.get("REPORTING_SQL_PORT", "3306") +REPORTING_SQL_DATABASE = os.environ.get("REPORTING_SQL_DATABASE", "myreportingdatabase") +REPORTING_SQL_USERNAME = os.environ.get("REPORTING_SQL_USERNAME") +REPORTING_SQL_PASSWORD = os.environ.get("REPORTING_SQL_PASSWORD") + + +AUDIT_SERVER = os.environ.get("AUDIT_SERVER", "127.0.0.1:27018") +AUDIT_REPLICASET = os.environ.get("AUDIT_REPLICASET", "rs4") +AUDIT_USERNAME = os.environ.get("MONGO_AUDIT_USERNAME") +AUDIT_PASSWORD = os.environ.get("MONGO_AUDIT_PASSWORD") +AUDIT_DATABASE = os.environ.get("AUDIT_DATABASE", "mydb") +AUDIT_COLLECTION = os.environ.get("AUDIT_COLLECTION", "myauditcollection") + + +SERVER_A = os.environ.get("SERVER_A", "127.0.0.1:27017") +SERVER_B = os.environ.get("SERVER_B", "127.0.0.1:27017") +SERVER_C = os.environ.get("SERVER_C", "127.0.0.1:27017") +REPLICASET_A = os.environ.get("REPLICASET_A", "rs0") +REPLICASET_B = os.environ.get("REPLICASET_B", "rs1") +REPLICASET_C = os.environ.get("REPLICASET_C", "rs2") + +USERNAME = os.environ.get("mongo_USERNAME") +PASSWORD = os.environ.get("mongo_PASSWORD") +DATABASE = os.environ.get("DATABASE", "mydb") +COLLECTION = os.environ.get("COLLECTION", "mycollection") + +ARC_MONGO_AUTHMECHANISM = os.environ.get("ARC_MONGO_AUTHMECHANISM", "SCRAM-SHA-1") +ARC_MONGO_AUTHSOURCE = os.environ.get("ARC_MONGO_AUTHSOURCE", "admin") +ARC_MONGO_READ_PREFERENCE = os.environ.get("ARC_MONGO_READ_PREFERENCE", "secondary") + + +REPORTING_AULDATALEAK_TABLENAME = os.environ.get( + "REPORTING_AULDATALEAK_TABLENAME", "auldata_leak" +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8547457 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,38 @@ +from unittest.mock import patch + +import pytest + + +@pytest.fixture(autouse=True) +def mock_mongo_client(): + mock_mongo_client = patch("sample_code.dao._base_mongo.MongoClient") + yield mock_mongo_client.start() + mock_mongo_client.stop() + + +@pytest.fixture(autouse=True) +def mock_usage_mongo_run_agg_query(): + mock_run_query = patch("sample_code.dao.audit.AuditDAO.run_aggregation_query") + yield mock_run_query.start() + mock_run_query.stop() + + +@pytest.fixture(autouse=True) +def mock_usage_mongo_run_query(): + mock_run_query = patch("sample_code.dao.usage.UsageDAO.run_query") + yield mock_run_query.start() + mock_run_query.stop() + + +@pytest.fixture(autouse=True) +def mock_report_mysql_client(): + mock_mysql_client = patch("sample_code.dao.reporting.create_engine") + yield mock_mysql_client.start() + mock_mysql_client.stop() + + +@pytest.fixture() +def mock_main_run_compare_on_node(): + mock_compare_on_node = patch("sample_code.main.Main.run_compare_on_node") + yield mock_compare_on_node.start() + mock_compare_on_node.stop() diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..d360597 --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,68 @@ +from datetime import date, timedelta + +from sample_code.dao.audit import AuditDAO + + +def test_get_subscribers(mock_usage_mongo_run_agg_query): + startDate = date.today() - timedelta(days=1) + endDate = date.today() + auditClient = AuditDAO("mongo-server.com", "mongo-replicaset") + + auditClient.get_subscribers(startDate, endDate) + assert mock_usage_mongo_run_agg_query.called_once() + assert mock_usage_mongo_run_agg_query.mock_calls[0].args[1] == [ + { + "$match": { + "$and": [ + { + "details": { + "$elemMatch": { + "state": "ADD", + "data.payload.payloads": { + "$elemMatch": { + "requestpayload.subscriptions": { + "$elemMatch": {"offerName": "MYOFFERNAME"} + } + } + }, + } + } + }, + { + "lastModifiedDate": { + "$gte": startDate, + "$lte": endDate, + } + }, + ] + } + }, + {"$unwind": {"path": "$details"}}, + { + "$match": { + "details.state": "ADD", + "details.data.payload.payloads": { + "$elemMatch": { + "requestpayload.subscriptions": { + "$elemMatch": {"offerName": "MYOFFERNAME"} + } + } + }, + } + }, + {"$unwind": {"path": "$details.data.payload.payloads"}}, + { + "$unwind": { + "path": "$details.data.payload.payloads.requestpayload.subscriptions" + } + }, + { + "$project": { + "_id": 0.0, + "ban": 1.0, + "subscriberId": "$details.data.payload.subscriberId", + "effectiveDate": "$details.data.payload.payloads.requestpayload.subscriptions.effectiveDate", + "expiryDate": "$details.data.payload.payloads.requestpayload.subscriptions.expiryDate", + } + }, + ] diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..9e7bed4 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,97 @@ +from unittest.mock import patch + +import pandas as pd +from freezegun import freeze_time + +from sample_code.main import Main + + +def test_compare(mock_main_run_compare_on_node): + def serialize_list_series(series): + return [dict(s) for s in series] + + data = pd.DataFrame( + [ + {"ban": 1, "subscriberId": 1}, + {"ban": 2, "subscriberId": 3}, + {"ban": 3, "subscriberId": 6}, + {"ban": 4, "subscriberId": 7}, + {"ban": 5, "subscriberId": 21}, + ] + ) + mainClient = Main() + mainClient.compare(data) + + assert len(mock_main_run_compare_on_node.mock_calls) == 3 + + assert mock_main_run_compare_on_node.mock_calls[0].args[0] == "A" + assert serialize_list_series( + mock_main_run_compare_on_node.mock_calls[0].args[1] + ) == [{"ban": 3, "subscriberId": 6}] + + assert mock_main_run_compare_on_node.mock_calls[1].args[0] == "B" + assert serialize_list_series( + mock_main_run_compare_on_node.mock_calls[1].args[1] + ) == [{"ban": 1, "subscriberId": 1}, {"ban": 4, "subscriberId": 7}] + + assert mock_main_run_compare_on_node.mock_calls[2].args[0] == "C" + assert serialize_list_series( + mock_main_run_compare_on_node.mock_calls[2].args[1] + ) == [{"ban": 2, "subscriberId": 3}, {"ban": 5, "subscriberId": 21}] + + +@freeze_time("2023-07-13 12:00:01") +@patch("sample_code.dao.usage.UsageDAO.get_subscriber_usage") +def test_run_compare_on_node__case_one_sub( + mock_get_subscriber_usage, mock_report_mysql_client +): + mock_get_subscriber_usage.return_value = pd.DataFrame( + [ + { + "extSubId": "1", + "MDN": "+2126666666", + "BAN": 1, + "start": "", + "end": "", + "bytesIn": 2048, + "bytesOut": 512, + }, + ] + ) + data = [ + { + "ban": 2, + "subscriberId": "3", + "effectiveDate": "2022-12-11T09:15:30Z", + "expiryDate": "2023-01-01T11:43:00Z", + } + ] + mainClient = Main() + mock_report_mysql_client.reset_mock() + mainClient.run_compare_on_node("A", data) + + assert len(mock_report_mysql_client.mock_calls) == 1 + assert ( + mock_report_mysql_client.mock_calls[0].args[0] + == "INSERT INTO auldata_leak (SUBSCRIBERID, MDN, BAN, USAGESTART, USAGEEND, TOTALMB, AUDITDATE) VALUES (1, +2126666666, 1, , , 2560, 2023-07-13 12:00:01)" + ) + + +@patch("sample_code.dao.usage.UsageDAO.get_subscriber_usage") +def test_run_compare_on_node__case_no_sub( + mock_get_subscriber_usage, mock_report_mysql_client +): + mock_get_subscriber_usage.return_value = pd.DataFrame([]) + data = [ + { + "ban": 2, + "subscriberId": "3", + "effectiveDate": "2022-12-11T09:15:30Z", + "expiryDate": "2023-01-01T11:43:00Z", + } + ] + mainClient = Main() + mock_report_mysql_client.reset_mock() + mainClient.run_compare_on_node("A", data) + + assert len(mock_report_mysql_client.mock_calls) == 0 diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..3182368 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,33 @@ +from sample_code.dao.reporting import ReportDAO + + +def test_process_data_for_insert(): + data = [[1, 3, "Test", "Hello world"], [1, 3, "Audit", "Rain man"]] + res = ReportDAO.process_data_for_insert(data) + + assert res == "(1, 3, Test, Hello world), (1, 3, Audit, Rain man)" + + +def test_insert_reporting_data(mock_report_mysql_client): + data = [[1, 3, "Test", "Hello world"], [1, 3, "Audit", "Rain man"]] + reportClient = ReportDAO() + mock_report_mysql_client.reset_mock() + + reportClient.insert_reporting_data(data) + assert len(mock_report_mysql_client.mock_calls) == 1 + assert ( + mock_report_mysql_client.mock_calls[0].args[0] + == "INSERT INTO auldata_leak (SUBSCRIBERID, MDN, BAN, USAGESTART, USAGEEND, TOTALMB, AUDITDATE) VALUES (1, 3, Test, Hello world), (1, 3, Audit, Rain man)" + ) + + +def test_clean_reporting_data(mock_report_mysql_client): + reportClient = ReportDAO() + mock_report_mysql_client.reset_mock() + + reportClient.clean_reporting_data() + assert len(mock_report_mysql_client.mock_calls) == 1 + assert ( + mock_report_mysql_client.mock_calls[0].args[0] + == "DELETE FROM auldata_leak WHERE AUDITDATE < DATE_SUB(NOW(), INTERVAL 1 MONTH)" + ) diff --git a/tests/test_usage.py b/tests/test_usage.py new file mode 100644 index 0000000..3d0e923 --- /dev/null +++ b/tests/test_usage.py @@ -0,0 +1,30 @@ +from datetime import date, timedelta + +from sample_code.dao.usage import UsageDAO + + +def test_get_subscriber_usage(mock_usage_mongo_run_query): + effDate = date.today() - timedelta(days=1) + expDate = date.today() + usageClient = UsageDAO("mongo-server.com", "mongo-replicaset") + + usageClient.get_subscriber_usage("1", effDate, expDate) + assert mock_usage_mongo_run_query.called_once() + assert mock_usage_mongo_run_query.mock_calls[0].args[1] == { + "$and": [ + {"end": {"$gte": effDate, "$lte": expDate}}, + {"extSubId": 1}, + {"usageType": "OVER"}, + {"$or": [{"bytesIn": {"$gt": 0}, "bytesOut": {"$gt": 0}}]}, + ] + } + assert mock_usage_mongo_run_query.mock_calls[0].args[2] == { + "_id": 0, + "extSubId": 1, + "MDN": 1, + "BAN": 1, + "start": 1, + "end": 1, + "bytesIn": 1, + "bytesOut": 1, + }