diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cda472a..e5e1182 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,10 @@ on: push: branches: ["master"] +concurrency: + group: "${{ github.workflow }}-${{ github.ref_name }}" + cancel-in-progress: "${{ github.ref_name != github.event.repository.default_branch }}" + jobs: create-semantic-tag: runs-on: "ubuntu-latest" @@ -62,3 +66,7 @@ jobs: uses: "./.github/workflows/release.yml" with: release_name: "${{ needs.create-semantic-tag.outputs.RELEASE_VERSION }}" + + deploy-docs: + needs: ["release-latest"] + uses: "./.github/workflows/doc.yml" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 0000000..6f190c1 --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,36 @@ +name: "Documentation" + +on: + workflow_call: + +jobs: + deploy: + permissions: + pages: "write" + id-token: "write" + environment: + name: "github-pages" + url: ${{ steps.deployment.outputs.page_url }} + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@v4" + + - name: "Set up Python" + uses: "actions/setup-python@v5" + with: + python-version: '3.x' + + - name: "Install requirements" + run: "make doc/prepare" + + - name: "Build site" + run: "make doc/build" + + - name: "Upload artifact" + uses: "actions/upload-pages-artifact@v3" + with: + path: "./site" + + - name: "Deploy to GitHub Page" + id: "deployment" + uses: "actions/deploy-pages@v4" diff --git a/.gitignore b/.gitignore index f161987..49d2d66 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ config/* !config/config.yml !config/models.yml -### Go template +### Go # Binaries for programs and plugins *.exe @@ -37,6 +37,116 @@ vendor/ # Go workspace file go.work +### Python + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Translations +*.mo +*.pot + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + ### VSCode .vscode diff --git a/Makefile b/Makefile index 444540f..75d1ecb 100644 --- a/Makefile +++ b/Makefile @@ -24,3 +24,4 @@ test/performance: include ./build/package/Makefile include ./build/ci/Makefile +include ./doc/build/Makefile diff --git a/README.md b/README.md index 4b3d830..6d870a1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ +
+ # Synthetic Data Values Generator (SDVG) +
+ [![Release][release-badge]][release-url] [![Pre-release][pre-release-badge]][pre-release-url] [![CI][actions-badge]][actions-url] @@ -7,31 +11,36 @@ [![Language][language-badge]][language-url] [![License][license-badge]][license-url] -[release-badge]: https://img.shields.io/github/v/release/tarantool/sdvg +[release-badge]: https://badgen.net/github/release/tarantool/sdvg/stable?color=pink [release-url]: https://github.com/tarantool/sdvg/releases/latest/ -[pre-release-badge]: https://img.shields.io/badge/pre--release-latest-orange +[pre-release-badge]: https://badgen.net/static/pre-release/latest/orange?color=purple [pre-release-url]: https://github.com/tarantool/sdvg/releases/tag/latest/ -[actions-badge]: https://img.shields.io/github/check-runs/tarantool/sdvg/master +[actions-badge]: https://badgen.net/github/checks/tarantool/sdvg [actions-url]: https://github.com/tarantool/sdvg/actions -[test-coverage-badge]: https://img.shields.io/coverallsCoverage/github/tarantool/sdvg?branch=master +[test-coverage-badge]: https://badgen.net/coveralls/c/github/tarantool/sdvg [test-coverage-url]: https://coveralls.io/github/tarantool/sdvg?branch=master -[language-badge]: https://img.shields.io/github/languages/top/tarantool/sdvg +[language-badge]: https://badgen.net/static/language/go/blue [language-url]: https://github.com/tarantool/sdvg/search?l=go -[license-badge]: https://img.shields.io/github/license/tarantool/sdvg +[license-badge]: https://badgen.net/github/license/tarantool/sdvg?color=cyan [license-url]: ./LICENSE -![scheme.png](asset/scheme.png) +
-## Language +## Documentation version +- [Multilingual web version](https://tarantool.github.io/sdvg/) (recommended) - **English** - [Русский](README.ru.md) +
+ ## Description SDVG (Synthetic Data Values Generator) is a tool for generating synthetic data. It supports various run modes, data types for generation, and output formats. +![scheme.png](./asset/scheme.png) + Run modes: - CLI - generate data, create configs, and validate them via the console; @@ -180,7 +189,7 @@ sdvg generate -h More information can be found in the [user guide](./doc/en/usage.md). -## Documentation +## Next Steps - [User Guide](./doc/en/usage.md) - [Developer Guide](./doc/en/contributing.md) diff --git a/README.ru.md b/README.ru.md index 9f0933c..e1adf6c 100644 --- a/README.ru.md +++ b/README.ru.md @@ -1,5 +1,9 @@ +
+ # Synthetic Data Values Generator (SDVG) +
+ [![Release][release-badge]][release-url] [![Pre-release][pre-release-badge]][pre-release-url] [![CI][actions-badge]][actions-url] @@ -7,31 +11,36 @@ [![Language][language-badge]][language-url] [![License][license-badge]][license-url] -[release-badge]: https://img.shields.io/github/v/release/tarantool/sdvg +[release-badge]: https://badgen.net/github/release/tarantool/sdvg/stable?color=pink [release-url]: https://github.com/tarantool/sdvg/releases/latest/ -[pre-release-badge]: https://img.shields.io/badge/pre--release-latest-orange +[pre-release-badge]: https://badgen.net/static/pre-release/latest/orange?color=purple [pre-release-url]: https://github.com/tarantool/sdvg/releases/tag/latest/ -[actions-badge]: https://img.shields.io/github/check-runs/tarantool/sdvg/master +[actions-badge]: https://badgen.net/github/checks/tarantool/sdvg [actions-url]: https://github.com/tarantool/sdvg/actions -[test-coverage-badge]: https://img.shields.io/coverallsCoverage/github/tarantool/sdvg?branch=master +[test-coverage-badge]: https://badgen.net/coveralls/c/github/tarantool/sdvg [test-coverage-url]: https://coveralls.io/github/tarantool/sdvg?branch=master -[language-badge]: https://img.shields.io/github/languages/top/tarantool/sdvg +[language-badge]: https://badgen.net/static/language/go/blue [language-url]: https://github.com/tarantool/sdvg/search?l=go -[license-badge]: https://img.shields.io/github/license/tarantool/sdvg +[license-badge]: https://badgen.net/github/license/tarantool/sdvg?color=cyan [license-url]: ./LICENSE -![scheme.png](asset/scheme.png) +
-## Язык +## Версия документации +- [Многоязычная веб-версия](https://tarantool.github.io/sdvg/ru/) (рекомендуется) - [English](README.md) - **Русский** +
+ ## Описание продукта SDVG (Synthetic Data Values Generator) — это инструмент для генерации синтетических данных. Он поддерживает различные форматы запуска, типы данных для генерации и форматы вывода. +![scheme.png](./asset/scheme.png) + Форматы запуска: - CLI - генерация данных, создание конфигураций и их валидация через консоль; @@ -180,7 +189,7 @@ sdvg generate -h Больше информации можно найти в [руководстве по эксплуатации](./doc/ru/usage.md). -## Документация +## Следующие шаги - [Руководство по эксплуатации](./doc/ru/usage.md) - [Руководство для разработчиков](./doc/ru/contributing.md) diff --git a/asset/favicon.png b/asset/favicon.png new file mode 100644 index 0000000..ca0dbf7 Binary files /dev/null and b/asset/favicon.png differ diff --git a/doc/assets/images/favicon.png b/doc/assets/images/favicon.png new file mode 120000 index 0000000..87020e0 --- /dev/null +++ b/doc/assets/images/favicon.png @@ -0,0 +1 @@ +../../../asset/favicon.png \ No newline at end of file diff --git a/doc/assets/images/scheme.png b/doc/assets/images/scheme.png new file mode 120000 index 0000000..eaffc12 --- /dev/null +++ b/doc/assets/images/scheme.png @@ -0,0 +1 @@ +../../../asset/scheme.png \ No newline at end of file diff --git a/doc/assets/stylesheets/sdvg.css b/doc/assets/stylesheets/sdvg.css new file mode 100644 index 0000000..d783bc7 --- /dev/null +++ b/doc/assets/stylesheets/sdvg.css @@ -0,0 +1,23 @@ +.hide-in-mkdocs { + display: none; +} + +.md-header__title { + margin-left: 0 !important; +} + +[data-md-color-scheme="slate"] { + --md-code-fg-color: #ffffff; + --md-code-bg-color: #333542; +} + +[data-md-color-scheme="default"] { + --md-code-bg-color: #ECEFF1; +} + +.md-typeset code { + padding: 0.2em 0.4em; + border-radius: 4px; + white-space: pre-wrap; + word-wrap: break-word; +} \ No newline at end of file diff --git a/doc/build/Makefile b/doc/build/Makefile new file mode 100644 index 0000000..93ad0f7 --- /dev/null +++ b/doc/build/Makefile @@ -0,0 +1,21 @@ +default: docs +all: default + +doc: doc/prepare doc/serve + +doc/prepare: + pip install -r ./doc/build/requirements.txt + +doc/serve: + mkdocs serve \ + --dev-addr "127.0.0.1:8000" \ + --open \ + --strict \ + --clean \ + -f ./mkdocs.yml + +.PHONY: doc/build +doc/build: + mkdocs build \ + --strict \ + --site-dir ./site diff --git a/doc/build/hooks.py b/doc/build/hooks.py new file mode 100644 index 0000000..59612f5 --- /dev/null +++ b/doc/build/hooks.py @@ -0,0 +1,33 @@ +import re + + +def replace_specific_links(markdown, *, page, config, files, **kwargs): + # Replace link from /CHANGELOG.md to page with changelog + markdown = re.sub( + r'\[([^]]+)](\(|:\s*).*CHANGELOG.md(\)|\s*)', + r'[\1]\2changelog.md\3', + markdown + ) + + # Replace link from /LICENSE to page with license + markdown = re.sub( + r'\[([^]]+)](\(|:\s*).*LICENSE(\)|\s*)', + r'[\1]\2license.md\3', + markdown + ) + + # Replace links for images + markdown = re.sub( + r'\[([^]]+)](\(|:\s*)(?:\.\./){2}asset/(.+\.(?:png|jpg))(\)|\s*)', + rf'[\1]\2assets/images/\3\4', + markdown + ) + + # Replace links for code + markdown = re.sub( + r'\[([^]]+)](\(|:\s*)(?:\.\./){2}(?!asset/)(.+)(\)|\s*)', + rf'[\1]\2{config.repo_url}/tree/master/\3\4', + markdown + ) + + return markdown diff --git a/doc/build/requirements.txt b/doc/build/requirements.txt new file mode 100644 index 0000000..b1b5223 --- /dev/null +++ b/doc/build/requirements.txt @@ -0,0 +1,6 @@ +mkdocs==1.6.1 +mkdocs-static-i18n==1.3.0 +mkdocs-material==9.6.15 +mkdocs-include-markdown-plugin==7.1.6 +mkdocs-simple-hooks==0.1.5 +pymdown-extensions==10.16 diff --git a/doc/en/changelog.md b/doc/en/changelog.md new file mode 120000 index 0000000..699cc9e --- /dev/null +++ b/doc/en/changelog.md @@ -0,0 +1 @@ +../../CHANGELOG.md \ No newline at end of file diff --git a/doc/en/contributing.md b/doc/en/contributing.md index c368c02..4f73389 100644 --- a/doc/en/contributing.md +++ b/doc/en/contributing.md @@ -24,6 +24,34 @@ To calculate code coverage, run `make test/cover`. To build the generator for your current OS, run `make build`. The resulting binaries will be in the `build/out` directory. +## Documentation + +To edit the documentation, you need to make changes for each supported language +within its corresponding directory in the [doc](../../doc) directory. + +To add a new section to the documentation, you must: + +1. Create a new `.md` file in the root of the [doc/en](../../doc/en) directory. +2. Add the file name to the `nav` section in the [mkdocs.yml](../../mkdocs.yml) configuration file. +3. Perform the same steps in the corresponding directories for each language if you need to support them. + Otherwise, the English version of the section will be displayed. + +To add support for a new language, you need to: + +1. Create a directory in the root of the [doc](../../doc) directory. The directory name must match + the language code (e.g., ru, en, ...) that you want to support. +2. Translate the content of all the sections that you want to support in the new language. + For everything to work correctly, the filenames of the translated documentation must match + the names in the `doc/en` directory. All untranslated files will be replaced with the English version. +3. Add the new language to the `plugins.i18n.languages` section in the `mkdocs.yml` configuration file, + where the `locale` property must match the name of the directory for the corresponding language. + +To locally check the documentation site's layout, you need to: + +1. Install the Python dependencies: `make doc/prepare`. +2. Run the local site hosting with the command `make doc/serve`. + The site will be available at [127.0.0.1:8000](http://127.0.0.1:8000). + ## Release To release a new version: diff --git a/doc/en/index.md b/doc/en/index.md new file mode 100644 index 0000000..3e71f61 --- /dev/null +++ b/doc/en/index.md @@ -0,0 +1,3 @@ +# Home + +{% include-markdown "../../README.md" rewrite-relative-urls=true %} diff --git a/doc/en/license.md b/doc/en/license.md new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/doc/en/license.md @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/doc/en/overview.md b/doc/en/overview.md index c3baa57..d94fe8f 100644 --- a/doc/en/overview.md +++ b/doc/en/overview.md @@ -1,5 +1,7 @@ # Goals and Standards Compliance +
+ ## Table of Contents - [Tool Benefits](#tool-benefits) @@ -7,6 +9,8 @@ - [VK Tech Values](#vk-tech-values) - [CNCF Standards Compliance](#cncf-standards-compliance) +
+ ## Tool Benefits With the rise of tools for working with large datasets (millions of rows) and large language models (LLMs), diff --git a/doc/en/usage.md b/doc/en/usage.md index 73c7d20..c9be61c 100644 --- a/doc/en/usage.md +++ b/doc/en/usage.md @@ -1,5 +1,7 @@ # User Guide +
+ ## Table of Contents - [Configuration](#configuration) @@ -14,6 +16,8 @@ - [Ignoring conflicts](#ignoring-conflicts) - [Continuing generation](#continuing-generation) +
+ ## Configuration SDVG uses two configuration files: the SDVG instance configuration file and the data generation configuration file. diff --git a/doc/ru/contributing.md b/doc/ru/contributing.md index 7d7efaf..0022720 100644 --- a/doc/ru/contributing.md +++ b/doc/ru/contributing.md @@ -24,6 +24,34 @@ Для запуска сборки генератора под текущую ОС используется команда `make build`. Результирующие бинарные файлы будут в папке `build/out`. +## Документация + +Для редактирования документации необходимо вносить правки для каждого поддерживаемого языка +внутри соответствующей ему директории в директории [doc](../../doc). + +Для добавления новой секции в документации необходимо: + +1. Создать новый `.md` файл в корне директории [doc/en](../../doc/en). +2. Добавить в конфигурационный файл [mkdocs.yml](../../mkdocs.yml) в секцию `nav` название файла. +3. Проделать аналогичные действия в соответствующих директориях для каждого языка, если их необходимо поддержать. + Иначе будет отображена английская версия секции. + +Для поддержки нового языка необходимо: + +1. Создать директорию в корне директории [doc](../../doc). Имя директории должно совпадать с кодом языка (ru, en, ...), + который вы хотите поддержать. +2. Перевести содержимое всех разделов, которые вы хотите поддержать в добавляемом языке. Для корректной работы + названия файлов с переведенной документацией должны совпадать с названиями из директории [doc/en](../../doc/en). + Все непереведенные файлы будут заменены английской версией. +3. Добавить в конфигурационный файл [mkdocs.yml](../../mkdocs.yml) в секцию `plugins.i18n.languages` новый язык, + где свойство `locale` должно совпадать с названием директории для соответствующего языка. + +Для того чтобы локально проверить вёрстку сайта с документацией необходимо: + +1. Скачать зависимости для python: `make doc/prepare`. +2. Запустить локальный хостинг сайта командой `make doc/serve`. + Сайт будет доступен по адресу [127.0.0.1:8000](http://127.0.0.1:8000). + ## Выпуск релиза Для выпуска нового релиза необходимо: diff --git a/doc/ru/index.md b/doc/ru/index.md new file mode 100644 index 0000000..23f0816 --- /dev/null +++ b/doc/ru/index.md @@ -0,0 +1,3 @@ +# Главная + +{% include-markdown "../../README.ru.md" rewrite-relative-urls=true %} diff --git a/doc/ru/overview.md b/doc/ru/overview.md index c958a3d..b595e8e 100644 --- a/doc/ru/overview.md +++ b/doc/ru/overview.md @@ -1,5 +1,7 @@ # Цели и соответствие стандартам +
+ ## Оглавление - [Польза инструмента](#польза-инструмента) @@ -7,6 +9,8 @@ - [Ценности VK Tech](#ценности-vk-tech) - [Соответствие стандартам CNCF](#соответствие-стандартам-cncf) +
+ ## Польза инструмента С появлением инструментов для работы с большими наборами данных (миллионы строк) и больших языковых моделей (LLM), diff --git a/doc/ru/usage.md b/doc/ru/usage.md index 0a44b75..7302a68 100644 --- a/doc/ru/usage.md +++ b/doc/ru/usage.md @@ -1,5 +1,7 @@ # Руководство по эксплуатации +
+ ## Оглавление - [Конфигурация](#конфигурация) @@ -14,6 +16,8 @@ - [Игнорирование конфликтов](#игнорирование-конфликтов) - [Продолжение генерации](#продолжение-генерации) +
+ ## Конфигурация SDVG использует два конфигурационных файла: файл конфигурации экземпляра SDVG и файл конфигурации генерации данных. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..9d14e9f --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,97 @@ +site_name: "SDVG" +site_url: "https://tarantool.github.io/sdvg" +repo_url: "https://github.com/tarantool/sdvg" +repo_name: "tarantool/sdvg" + +### BUILD ### + +docs_dir: "doc" +exclude_docs: | + build/* + +validation: + omitted_files: "warn" + absolute_links: "warn" + unrecognized_links: "warn" + anchors: "warn" + +nav: + - "index.md" + - "usage.md" + - "contributing.md" + - "overview.md" + - "changelog.md" + - "license.md" + +extra_css: + - "assets/stylesheets/sdvg.css" + +### APPEARANCE ### + +theme: + name: "material" + logo: "assets/images/favicon.png" + favicon: "assets/images/favicon.png" + icon: + repo: "fontawesome/brands/github" + palette: + - media: "(prefers-color-scheme)" + toggle: + name: "Switch to light mode" + icon: "material/brightness-auto" + - media: "(prefers-color-scheme: light)" + scheme: "default" + toggle: + name: "Switch to dark mode" + icon: "material/brightness-7" + - media: "(prefers-color-scheme: dark)" + scheme: "slate" + toggle: + name: "Switch to system preference" + icon: "material/brightness-4" + + features: + - "content.code.copy" # copy button for code sections + - "navigation.footer" + - "navigation.path" + - "navigation.indexes" + - "navigation.sections" + - "navigation.expand" + - "toc.follow" + - "search.highlight" + +extra: + generator: false # disable footer note 'Made with Material for MkDocs' + +markdown_extensions: + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.superfences: + - pymdownx.inlinehilite: + - attr_list: + - codehilite: + - toc: + permalink: true + slugify: !!python/object/apply:pymdownx.slugs.slugify # for cyrillic + kwds: + case: "lower" + +plugins: + - include-markdown: + - mkdocs-simple-hooks: + hooks: + on_page_markdown: "doc.build.hooks:replace_specific_links" + - search: + - i18n: + docs_structure: "folder" + fallback_to_default: true + reconfigure_search: true + reconfigure_material: true + languages: + - locale: "en" + name: "English" + default: true + - locale: "ru" + name: "Русский"