commit f4cddc3e3e8c5451ebca832d03286c3165b0bd12 Author: csasq Date: Sat Feb 1 13:51:06 2025 +0300 Разработана утилита для многопоточного сжатия файлов WebAssembly diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..633b1f1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/config.ini diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fe463e0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,408 @@ +Attribution-NonCommercial 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-NonCommercial 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-NonCommercial 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. NonCommercial means not primarily intended for or directed towards + commercial advantage or monetary compensation. For purposes of + this Public License, the exchange of the Licensed Material for + other material subject to Copyright and Similar Rights by digital + file-sharing or similar means is NonCommercial provided there is + no payment of monetary compensation in connection with the + exchange. + + j. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + k. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + l. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce, reproduce, and Share Adapted Material for + NonCommercial purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database for NonCommercial purposes + only; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..31a2493 --- /dev/null +++ b/README.md @@ -0,0 +1,171 @@ +# Wasm Encoding Tool + +![Static Badge](https://img.shields.io/badge/Powered%20by-csasq-blue) +![Static Badge](https://img.shields.io/badge/Python-3.9%2B-blue) +![Static Badge](https://img.shields.io/badge/License-CC%20BY--NC%204.0-blue) + +Утилита для сжатия файлов WebAssembly (Wasm) с использованием наиболее распространенных кодировок, поддерживаемых браузерами. Данный подход позволяет заранее подготовить сжатые файлы для отдачи их веб-сервером. + +## Преимущества + +Ниже приведено сравнение размера Wasm без сжатия и с применением максимального уровня сжатия каждым из поддерживаемых алгоритмов. + +Размер исходного файла Wasm — 8.0M. + +| Алгоритм сжатия | Размер | Коэффициент сжатия | Степень сжатия | +|-----------------|-------:|-------------------:|---------------:| +| Brotli | 2.5M | 3.2 | 68.75% | +| Zstandard | 2.6M | 3.08 | 67.5% | +| Gzip | 3.1M | 2.58 | 61.25% | +| Deflate | 3.1M | 2.58 | 61.25% | + +Как следует из таблицы, в лучшем случае сжатый файл имеет размер приблизительно в 3,2 раза меньше, чем исходный. + +## Рекомендации + +1. Следите за пересечением относительных путей, если меняете предустановленные значения конфигурации. +2. Протестируйте работу утилиты с различным количеством параллельных потоков. Зачастую, вы можете запускать сжатие с неограниченно большим значением параметра `max_threads`. В лучшем случае получится добиться сжатия каждого файла каждым алгоритмом в отдельном потоке. В худшем случае процесс будет убит средствами OOM Killer. + +## Конфигурация + +Файл конфигурации следует расположить в `./config.ini` относительно рабочей директории утилиты. + +Ниже приведен пример конфигурации со всеми возможными разделами и параметрами. Все пути за исключением параметра `Main.base_path` можно указывать как абсолютными, так и относительными. + +```ini +[Main] +base_path = /opt/my-project +max_threads = 4 +log_level = info + +[Input] +path = ./source + +[Output] +path = ./public +dir_mode = 755 +file_mode = 644 + +[Output.Wasm.Source] +enabled = true +path = ./wasm/src + +[Output.Wasm.Brotli] +enabled = true +path = ./wasm/br +level = 11 + +[Output.Wasm.Zstandard] +enabled = true +path = ./wasm/zstd +level = 22 + +[Output.Wasm.Gzip] +enabled = true +path = ./wasm/gzip +level = 9 + +[Output.Wasm.Deflate] +enabled = true +path = ./wasm/deflate +level = 9 + +[Output.Other] +enabled = true +path = . + +[Backup] +enabled = true +path = ./backup +``` + +### Раздел `Main` + +Содержит основные параметры утилиты. + +- `base_path` — базовый путь для построения относительных путей; +- `max_threads` — максимальное количество параллельных потоков; +- `log_level` — уровень логирования (`critical`, `fatal`, `error`, `warn`, `warning`, `info`, `debug`, `notset`). + +### Раздел `Input` + +Содержит параметры входной директории. Относительные пути строятся от `Main.base_path`. + +- `path` — путь до исходников веб-приложения. + +### Раздел `Output` + +Содержит параметры выходной директории. Относительные пути строятся от `Main.base_path`. + +- `path` — путь до исходников веб-приложения после сжатия; +- `dir_mode` — режим доступа к директориям; +- `file_mode` — режим доступа файлам. + +### Раздел `Output.Wasm.Source` + +Содержит параметры копирования исходников Wasm. Относительные пути строятся от `Output.path`. + +- `enabled` — включить копирование исходников Wasm; +- `path` — путь до исходников Wasm. + +### Разделы `Output.Wasm.Brotli`, `Output.Wasm.Zstandard`, `Output.Wasm.Gzip`, `Output.Wasm.Deflate` + +Содержит параметры сжатия исходников Wasm. Относительные пути строятся от `Output.path`. + +- `enabled` — включить сжатие исходников Wasm алгоритмом; +- `path` — путь до сжатых исходников; +- `level` — уровень сжатия: + - Brotli — с `1` по `11`, + - Zstandard — с `1` по `22`, + - Gzip — с `1` по `9`, + - Deflate — с `1` по `9`. + +### Раздел `Output.Other` + +Содержит параметры копирования прочих исходников. Относительные пути строятся от `Output.path`. + +- `enabled` — включить копирование прочих файлов; +- `path` — путь до прочих файлов. + +### Раздел `Backup` + +Содержит параметры резервной копии. Относительные пути строятся от `Main.base_path`. + +- `enabled` — включить создание резервной копии; +- `path` — путь до резервной копии. + +## Конфигурация Nginx + +Ниже приведен пример конфигурации Nginx для корректной отдачи файлов Wasm с указанием заголовков `Content-Type` и `Content-Length`. + +Обратите внимание на порядок элементов в `map $http_accept_encoding $wasm_accept_encoding`. Изменив этот порядок, вы можете повлиять на приоритет применения алгоритмов сжатия. + +```nginx +map $http_accept_encoding $wasm_accept_encoding { + "~br" "br"; + "~zstd" "zstd"; + "~gzip" "gzip"; + "~deflate" "deflate"; + default "src"; +} + +map $wasm_accept_encoding $wasm_content_encoding { + "br" "br"; + "zstd" "zstd"; + "gzip" "gzip"; + "deflate" "deflate"; + default ""; +} + +root /opt/my-project/public; + +location / { + try_files $uri /index.html; +} + +location ~* \.wasm$ { + try_files /wasm/$wasm_accept_encoding/$uri =404; + default_type application/wasm; + add_header "Content-Encoding" $wasm_content_encoding; +} +``` diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..86966d1 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +from .main import run diff --git a/app/algorithms.py b/app/algorithms.py new file mode 100644 index 0000000..56caa94 --- /dev/null +++ b/app/algorithms.py @@ -0,0 +1,44 @@ +import gzip +import zlib + +import brotli +import zstandard + + +def compress_with_brotli( + data: bytes, + level: int, +): + return brotli.compress( + data, + quality=level, + ) + + +def compress_with_zstandard( + data: bytes, + level: int, +): + return zstandard.ZstdCompressor( + level=level, + ).compress(data) + + +def compress_with_gzip( + data: bytes, + level: int, +): + return gzip.compress( + data, + compresslevel=level, + ) + + +def compress_with_deflate( + data: bytes, + level: int, +): + return zlib.compress( + data, + level=level, + ) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..fabc161 --- /dev/null +++ b/app/main.py @@ -0,0 +1,246 @@ +from concurrent.futures import ThreadPoolExecutor, Future +import logging +from pathlib import PurePath, Path +import shutil + +import config + +from .algorithms import compress_with_brotli, compress_with_zstandard, compress_with_gzip, compress_with_deflate +from .utils import get_paths, copy_paths, log + + +logging.basicConfig( + level=config.Main.log_level, +) +logger = logging.getLogger('WasmEncodingTool') + +config.Output.Wasm.Brotli.compress = compress_with_brotli +config.Output.Wasm.Zstandard.compress = compress_with_zstandard +config.Output.Wasm.Gzip.compress = compress_with_gzip +config.Output.Wasm.Deflate.compress = compress_with_deflate + +algorithms = [ + config.Output.Wasm.Brotli, + config.Output.Wasm.Zstandard, + config.Output.Wasm.Gzip, + config.Output.Wasm.Deflate, +] + + +def backup(): + @log( + logger=logger, + start_text='Backup', + success_text='Success backup', + ) + def run(): + shutil.rmtree( + config.Backup.path, + ignore_errors=True, + ) + shutil.copytree( + config.Output.path, + config.Backup.path, + ) + + if not config.Backup.enabled: + logger.info('Backup disabled') + return + if not Path(config.Output.path).exists(): + logger.info('Backup skipped (output path don\'t exists)') + return + + run() + + +def copy_wasm_source( + input_wasm_paths: list[PurePath], +): + @log( + logger=logger, + start_text='Copy wasm source', + success_text='Success copy wasm source', + ) + def run(): + copy_paths( + from_dir_path=config.Input.path, + to_dir_path=config.Output.Wasm.Source.path, + paths=input_wasm_paths, + ) + + if not config.Output.Wasm.Source.enabled: + logger.info('Copy wasm source disabled') + return + if not len(input_wasm_paths): + logger.info('Copy wasm source skipped (input path don\'t exists)') + return + + run() + + +def copy_other_source( + input_other_paths: list[PurePath], +): + @log( + logger=logger, + start_text='Copy other source', + success_text='Success copy other source', + ) + def run(): + copy_paths( + from_dir_path=config.Input.path, + to_dir_path=config.Output.Other.path, + paths=input_other_paths, + ) + + if not config.Output.Other.enabled: + logger.info('Copy other source disabled') + return + if not len(input_other_paths): + logger.info('Copy other source skipped (input path don\'t exists)') + return + + run() + + +def compress_file_with_algorithm( + file_path: PurePath, + algorithm: config.Algorithm, +): + @log( + logger=logger, + start_text='Compress %s with %s' % ( + file_path, + algorithm.name, + ), + success_text='Success compress %s with %s' % ( + file_path, + algorithm.name, + ), + ) + def run(): + Path(algorithm.path).mkdir( + parents=True, + exist_ok=True, + ) + with ( + Path(config.Input.path, file_path).open('rb') as file_in, + Path(algorithm.path, file_path).open('wb') as file_out, + ): + file_out.write(algorithm.compress( + data=file_in.read(), + level=algorithm.level, + )) + + run() + + +def compress( + executor: ThreadPoolExecutor, + input_wasm_paths: list[PurePath], +): + @log( + logger=logger, + start_text='Compress', + success_text='Success compress', + ) + def run(): + futures: list[Future] = [] + for algorithm in algorithms: + if not algorithm.enabled: + logger.info('Skipped compress with %s' % algorithm.name) + continue + for path in input_wasm_paths: + future = executor.submit( + compress_file_with_algorithm, + file_path=path, + algorithm=algorithm, + ) + futures.append(future) + for future in futures: + future.result() + + if not len(input_wasm_paths): + logger.info('Compress skipped (input path don\'t exists)') + return + + run() + + +def run_multithreading_tasks( + input_wasm_paths: list[PurePath], + input_other_paths: list[PurePath], +): + @log( + logger=logger, + start_text='Run multithreading tasks', + success_text='Success run multithreading tasks', + ) + def run(): + shutil.rmtree( + config.Output.path, + ignore_errors=True, + ) + with ThreadPoolExecutor( + max_workers=config.Main.max_threads, + ) as executor: + executor.submit( + copy_wasm_source, + input_wasm_paths=input_wasm_paths, + ) + executor.submit( + copy_other_source, + input_other_paths=input_other_paths, + ) + compress( + executor=executor, + input_wasm_paths=input_wasm_paths, + ) + + run() + + +def change_mode(): + @log( + logger=logger, + start_text='Change mode', + success_text='Success change mode', + ) + def run(): + for path in Path(config.Output.path).rglob('*'): + if path.is_dir(): + path.chmod(config.Output.dir_mode) + elif path.is_file(): + path.chmod(config.Output.file_mode) + path.chmod(config.Output.dir_mode) + + if not Path(config.Output.path).exists(): + logger.info('Change mode skipped (output path don\'t exists)') + return + + run() + + +def run(): + @log( + logger=logger, + start_text='Start', + success_text='Done', + ) + def run(): + backup() + run_multithreading_tasks( + input_wasm_paths=get_paths( + from_dir_path=config.Input.path, + pattern='*.wasm', + check_function=lambda path: path.is_file(), + ), + input_other_paths=get_paths( + from_dir_path=config.Input.path, + pattern='*', + check_function=lambda path: path.is_file() and path.suffix != '.wasm', + ), + ) + change_mode() + + run() diff --git a/app/utils.py b/app/utils.py new file mode 100644 index 0000000..cb93f3a --- /dev/null +++ b/app/utils.py @@ -0,0 +1,54 @@ +from logging import Logger +from pathlib import PurePath, Path +import shutil +from time import perf_counter +from typing import Callable + + +def get_paths( + from_dir_path: PurePath, + pattern: str, + check_function: Callable = lambda path: True, +) -> list[PurePath]: + from_dir_path = Path(from_dir_path) + return [ + PurePath(path.relative_to(from_dir_path)) + for path in from_dir_path.rglob(pattern) + if check_function(path) + ] + + +def copy_paths( + from_dir_path: PurePath, + to_dir_path: PurePath, + paths: list[PurePath], +): + for path in paths: + to_file_path = Path(to_dir_path) / path + to_file_path.parent.mkdir( + parents=True, + exist_ok=True, + ) + shutil.copy2( + Path(from_dir_path) / path, + to_file_path, + ) + + +def log( + logger: Logger, + start_text: str, + success_text: str, +): + def decorator(func): + def wrapper(*args, **kwargs): + logger.info(start_text) + start_time = perf_counter() + func() + end_time = perf_counter() + logger.info('%s (%f s)' % ( + success_text, + end_time - start_time, + )) + return wrapper + return decorator diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..8bcf0e2 --- /dev/null +++ b/config/__init__.py @@ -0,0 +1,2 @@ +from .main import Main, Input, Output, Backup +from .models import Algorithm diff --git a/config/levels.py b/config/levels.py new file mode 100644 index 0000000..2306dc3 --- /dev/null +++ b/config/levels.py @@ -0,0 +1,13 @@ +from logging import CRITICAL, FATAL, ERROR, WARNING, INFO, DEBUG, NOTSET + + +levels = { + 'CRITICAL': CRITICAL, + 'FATAL': FATAL, + 'ERROR': ERROR, + 'WARN': WARNING, + 'WARNING': WARNING, + 'INFO': INFO, + 'DEBUG': DEBUG, + 'NOTSET': NOTSET, +} diff --git a/config/main.py b/config/main.py new file mode 100644 index 0000000..10c06c4 --- /dev/null +++ b/config/main.py @@ -0,0 +1,186 @@ +import configparser +from pathlib import PurePath +from typing import Callable + +from .levels import levels +from .models import Algorithm + + +config = configparser.RawConfigParser() +config.read( + filenames='./config.ini', + encoding='utf-8', +) + + +class Main: + base_path = PurePath(config.get( + section='Main', + option='base_path', + )) + max_threads = config.getint( + section='Main', + option='max_threads', + fallback=4, + ) + log_level = levels.get( + config.get( + section='Main', + option='log_level', + fallback='info', + ).upper() + ) + + +class Input: + path = PurePath(config.get( + section='Input', + option='path', + )) + + +class Output: + path = PurePath(config.get( + section='Output', + option='path', + )) + dir_mode = int(config.get( + section='Output', + option='dir_mode', + fallback='755', + ), 8) + file_mode = int(config.get( + section='Output', + option='file_mode', + fallback='644', + ), 8) + + class Wasm: + class Source: + enabled = config.getboolean( + section='Output.Wasm.Source', + option='enabled', + fallback=True, + ) + path = PurePath(config.get( + section='Output.Wasm.Source', + option='path', + fallback=PurePath('.', 'wasm', 'src'), + )) + + class Brotli(Algorithm): + compress: Callable + + name = 'Brotli' + enabled = config.getboolean( + section='Output.Wasm.Brotli', + option='enabled', + fallback=True, + ) + path = PurePath(config.get( + section='Output.Wasm.Brotli', + option='path', + fallback=PurePath('.', 'wasm', 'br'), + )) + level = config.getint( + section='Output.Wasm.Brotli', + option='level', + fallback=11, + ) + + class Zstandard(Algorithm): + compress: Callable + + name = 'Zstandard' + enabled = config.getboolean( + section='Output.Wasm.Zstandard', + option='enabled', + fallback=True, + ) + path = PurePath(config.get( + section='Output.Wasm.Zstandard', + option='path', + fallback=PurePath('.', 'wasm', 'zstd'), + )) + level = config.getint( + section='Output.Wasm.Zstandard', + option='level', + fallback=22, + ) + + class Gzip(Algorithm): + compress: Callable + + name = 'Gzip' + enabled = config.getboolean( + section='Output.Wasm.Gzip', + option='enabled', + fallback=True, + ) + path = PurePath(config.get( + section='Output.Wasm.Gzip', + option='path', + fallback=PurePath('.', 'wasm', 'gzip'), + )) + level = config.getint( + section='Output.Wasm.Gzip', + option='level', + fallback=9, + ) + + class Deflate(Algorithm): + compress: Callable + + name = 'Deflate' + enabled = config.getboolean( + section='Output.Wasm.Deflate', + option='enabled', + fallback=True, + ) + path = PurePath(config.get( + section='Output.Wasm.Deflate', + option='path', + fallback=PurePath('.', 'wasm', 'deflate'), + )) + level = config.getint( + section='Output.Wasm.Deflate', + option='level', + fallback=9, + ) + + class Other: + enabled = config.getboolean( + section='Output.Other', + option='enabled', + fallback=True, + ) + path = PurePath(config.get( + section='Output.Other', + option='path', + fallback=PurePath('.'), + )) + + +class Backup: + enabled = config.getboolean( + section='Backup', + option='enabled', + fallback=True, + ) + path = PurePath(config.get( + section='Backup', + option='path', + fallback=PurePath(Main.base_path, 'backup'), + )) + + +Input.path = Main.base_path / Input.path +Output.path = Main.base_path / Output.path +Backup.path = Main.base_path / Backup.path + +Output.Wasm.Source.path = Output.path / Output.Wasm.Source.path +Output.Wasm.Brotli.path = Output.path / Output.Wasm.Brotli.path +Output.Wasm.Zstandard.path = Output.path / Output.Wasm.Zstandard.path +Output.Wasm.Gzip.path = Output.path / Output.Wasm.Gzip.path +Output.Wasm.Deflate.path = Output.path / Output.Wasm.Deflate.path +Output.Other.path = Output.path / Output.Other.path diff --git a/config/models.py b/config/models.py new file mode 100644 index 0000000..2662762 --- /dev/null +++ b/config/models.py @@ -0,0 +1,13 @@ +from pathlib import PurePath +from typing import Callable, Type + + +class BaseAlgorithm: + compress: Callable + name: str + enabled: bool + path: PurePath + level: int + + +Algorithm = Type[BaseAlgorithm] diff --git a/main.py b/main.py new file mode 100644 index 0000000..540380a --- /dev/null +++ b/main.py @@ -0,0 +1,5 @@ +import app + + +if __name__ == '__main__': + app.run() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6266ef6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +brotli +zstandard