First commit

.gitignore

@@ -0,0 +1,194 @@
+## GitIgnore Template
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+
+## Node.js
+
+node_modules
+
+## OS specific 
+
+## Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+### Additional Gitignore Stuff:
+data/*
+**/data/*
+**/local/*
+**/mlruns/*
+**/mlruns
+**/output/*
+**/output/*.csv
+!**/output/.gitkeep
+!**/data/.gitkeep

README.md

@@ -0,0 +1,15 @@
+## Important note: 
+No data or output should be uploaded to this repo. Please make use of .gitignore template in the root directory if you have folder/directory containing dataset. The content in folder/directory currently being ignored from git push are data/ and output/, recursively.
+
+## Configure Hooks
+
+If this is the first time you are using Repo with this Template you have to setup the Git Hooks dir.
+
+To set up the hooks dir globally run `git config --global core.hooksPath ./.hooks/`
+
+To set up the hooks for only this Repo run `git config core.hooksPath ./.hooks/`
+
+## Please enter your general Project description here
+
+## If you don't need all folder feel free to delete them
+

app/proto-docs/.gitignore

@@ -0,0 +1,201 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+data/*
+!data/.gitkeep
+
+logs/*
+!logs/.gitkeep
+
+# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode

app/proto-docs/Makefile

@@ -0,0 +1,21 @@
+lint-check: 
+	@poetry run ruff check src/ tests/
+
+lint-fix:
+	@poetry run ruff --fix src/ tests/
+
+format-check:
+	@poetry run black --check --diff src/ tests/
+
+format-fix:
+	@poetry run black src/ tests/
+
+statictypes-check:
+	@poetry run pyright src/ tests/
+
+run-tests:
+	@poetry run pytest
+
+complete-check: lint-check format-check statictypes-check run-tests
+complete-fix: lint-fix format-fix
+

app/proto-docs/README.md

@@ -0,0 +1,24 @@
+# Introduction
+
+This repository is used as a prototype template for setting up new Python project.
+
+The documentation in `docs` contains the recommended practices.
+
+# Installation
+
+## Requirements
+
+- Install [pyenv](https://github.com/pyenv/pyenv#installation) to manage the Python versions
+- Install [Poetry](https://python-poetry.org/docs/#installation) to manage the dependencies
+
+Run the following command in this project root folder to install the dependencies:
+
+```bash
+poetry install
+```
+
+Or you can install the dependencies with `pip`:
+
+```bash
+pip install -r requirements.txt
+```

app/proto-docs/docs/code_quality.md

@@ -0,0 +1,86 @@
+# Formatting
+
+`Black` is recommended to format the Python files. It is a highly opinionated formatter that follows PEP8. 
+
+## Typical Workflow
+
+1. Add [`black`](https://pypi.org/project/black/) python library as dev dependency in the virtual environment. Pin down the version if necessary.
+    
+    === "Poetry"
+
+        ```bash
+        poetry add -G dev black
+        ```
+
+    === "Conda"
+
+        ```bash
+        conda install black
+        ```
+
+2. Install `Black` vscode extension as specified in [Computer Setup](./0_computer_setup.md)
+
+3. Enable format on save in your editor
+
+## Additional settings
+`Black` can be customized through `pyproject.toml`
+
+`Black` settings are explained in [here](https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html).
+
+# Linting
+
+`Ruff` is recommended to lint the Python files. 
+
+## Typical Workflow
+
+1. Add [`ruff`](https://pypi.org/project/ruff/) python library as dev dependency in the virtual environment. Pin down the version if necessary.
+
+    === "Poetry"
+
+        ```bash
+        poetry add -G dev ruff
+        ```
+
+    === "Conda"
+
+        ```bash
+        conda install ruff
+        ```
+
+2. Install `Ruff` vscode extension as specified in [Computer Setup](./0_computer_setup.md)
+
+## Additional setting
+
+`Ruff` can be customized through `pyproject.toml`
+
+`Ruff` settings are explained in [here](https://beta.ruff.rs/docs/configuration/).
+
+
+# Type checking
+
+`Pyright` is recommended to type check all python files in the project.
+
+## Typical Worklow
+
+1. Add [`pyright`](https://pypi.org/project/pyright/) python library as dev dependency in the virtual environment. Pin down the version if necessary.
+
+    === "Poetry"
+
+        ```bash
+        poetry add -G dev pyright
+        ```
+
+    === "Conda"
+
+        ```bash
+        conda install pyright
+        ```
+
+2. Install `Python` vscode extension as specified in [Computer Setup](./0_computer_setup.md)
+
+
+## Additional setting
+
+`Pyright` can be customized through `pyproject.toml`
+
+`Pyright` settings are explained in [here](https://github.com/microsoft/pyright/blob/main/docs/configuration.md)

app/proto-docs/docs/computer_setup.md

@@ -0,0 +1,101 @@
+# Visual Studio Code setup
+
+1.  Recommended vscode extensions for Python Development
+
+    - [`Python`](https://marketplace.visualstudio.com/items?itemName=ms-python.python)
+    - [`Black`](https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter)
+    - [`Ruff`](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff)
+    - [`Jupyter`](https://marketplace.visualstudio.com/items?itemName=ms-toolsai.jupyter)
+    - [`autoDocstring`](https://marketplace.visualstudio.com/items?itemName=njpwerner.autodocstring)
+    - [`Even Better TOML`](https://marketplace.visualstudio.com/items?itemName=tamasfe.even-better-toml)
+    - [`YAML`](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml)
+    - [`Remote Development`](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack)
+    - [`Docker`](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker)
+    - [`GitLens`](https://marketplace.visualstudio.com/items?itemName=eamodio.gitlens)
+
+2.  Enable strict typehints in the setting
+
+    ```json
+    {
+      "python.analysis.typeCheckingMode": "strict",
+      "python.analysis.inlayHints.pytestParameters": true,
+      "python.analysis.inlayHints.functionReturnTypes": true
+    }
+    ```
+
+3.  Format using `Black` and Lint using `Ruff` on save
+
+    !!! warning
+
+        Set all these settings to false for existing projects that never use `Black` or `Ruff` before.
+
+    ```json
+    {
+      "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+          "source.organizeImports": true,
+          "source.fixAll": true
+        }
+      },
+      "python.formatting.provider": "none",
+      "notebook.formatOnSave.enabled": true
+    }
+    ```
+
+4.  Test using `pytest`
+
+    ```json
+    {
+      "python.testing.pytestEnabled": true,
+      "python.testing.pytestArgs": ["tests"]
+    }
+    ```
+
+5.  Use `google` documentation style
+
+    ```json
+    {
+      "autoDocstring.docstringFormat": "google"
+    }
+    ```
+
+6.  Guides from the internet
+
+    - [Python in Visual Studio](https://code.visualstudio.com/docs/languages/python)
+
+# Virtual Environment Management
+
+Pyenv and Poetry is typically used for managing Python package, while Conda is more popular in Data Science project.
+
+Both features virtual environment creation, but Poetry offers finer details on installing packages for production or development, publishing the Python package.
+
+1.  **Both** [`Pyenv`](https://github.com/pyenv/pyenv) **AND** [`Poetry`](https://github.com/python-poetry/poetry/)
+
+    !!! note "Pyenv Installation"
+    Pyenv installation may requires executing more than one line in terminal, you also need to set your shell profile (`.zshrc` or `.bash_profile`).
+
+        Read carefully [here](https://github.com/pyenv/pyenv#installation)
+
+    !!! note "Poetry Installation"
+
+        If you have old Poetry version (<1.2), you need to uninstall it and install the latest version.
+
+2.  **Either** [Conda `Miniforge` **OR** `Mambaforge`](https://github.com/conda-forge/miniforge)
+
+3.  [Docker](https://www.docker.com/)
+
+# Gitignore
+
+Download gitignore template from [gitignore.io](https://www.toptal.com/developers/gitignore/).
+
+[`Python` and `visualstudiocode`](https://www.toptal.com/developers/gitignore/api/python,visualstudiocode) are recommended arguments for starter.
+
+## Alternative Way
+
+You may create command line function `gi` to download gitignore file. Follow the guide in [here](https://docs.gitignore.io/install/command-line). Then, you can the command in the project root folder.
+
+```bash
+gi python,visualstudiocode >> .gitignore
+```

app/proto-docs/docs/documentation.md

@@ -0,0 +1,37 @@
+# Introduction
+
+[`MkDocs`](https://www.mkdocs.org/) is a static site generator that's geared towards building project documentation. Documentation source files are written in `Markdown`, and configured with a single `YAML` configuration file.
+
+# Installation
+
+`MkDocs` is a Python library and there are also some extras recommended plugins to be installed.
+
+1. [`mkdocs-material`](https://pypi.org/project/mkdocs-material/) - Theme
+2. [`mkdocstrings`](https://pypi.org/project/mkdocstrings/) - Auto generate documentation from docstrings
+3. [`mkdocs-jupyter`](https://pypi.org/project/mkdocs-jupyter/) - Embed Jupyter notebook in the documentation
+
+=== "Poetry"
+
+    ```bash
+    poetry add -G doc mkdocs mkdocs-material "mkdocstrings[python]" mkdocs-jupyter
+    ```
+
+=== "Conda"
+
+    ```bash
+    conda install mkdocs mkdocs-material mkdocstrings mkdocs-jupyter
+    ```
+
+# Workflow
+
+1. Install mkdocs and the plugins as above
+
+2. Run `mkdocs new .` in the project root folder. It will create `mkdocs.yml` and `docs` folder
+
+3. Edit `mkdocs.yml` to configure the site
+
+4. Write the documentation in `docs` folder
+
+# References:
+
+- [MkDocs docs](https://www.mkdocs.org/)

app/proto-docs/docs/index.md

@@ -0,0 +1,41 @@
+# Introduction
+
+This documentation contains Python recommended practice. It is intended to be used as a reference for Python developers.
+
+## TODO Project:
+
+1. Deploy this documentation to Azure Static Web Apps. Set it private for Evonik users and ensure the url page is fixed so it is easy to be share and bookmarked.
+2. Create Python template with [`cookiecutter`](https://cookiecutter.readthedocs.io/en/latest/) and possibly [`cruft`](https://cruft.github.io/cruft/)
+
+   References:
+
+   - [Introducing a project template for modern Python packages](https://blog.wolt.com/engineering/2022/08/11/project-template-for-modern-python-packages/)
+
+## TODO Documentation Contents:
+
+1. Dockerize the project
+
+   References:
+
+   - https://pythonspeed.com/docker/
+
+2. Setup environment variables thorugh `.env`
+3. CI / CD with Azure Pipelines
+4. Testing with pytest, coverage, mock, and tox
+
+   References:
+
+   - https://cjolowicz.github.io/posts/hypermodern-python-02-testing/
+
+5. Makefile / pre-commit
+
+   References:
+
+   - https://earthly.dev/blog/python-makefile/
+   - https://pre-commit.com/
+
+6. Logging with structlog instead?
+
+   References:
+
+   - https://www.structlog.org/en/stable/

app/proto-docs/docs/logging.md

@@ -0,0 +1,26 @@
+# Introduction
+
+Logging is a very important part of any application. It allows you to see what is happening in your application, and is especially useful when debugging.
+
+Logging is already built in `Python` though `logging` library, and we may add `rich` library to make it more beautiful.
+
+=== "Poetry"
+
+    ```bash
+    poetry add rich
+    ```
+
+=== "Conda"
+
+    ```bash
+    conda install rich
+    ```
+
+There are several ways to write logging configuration. You can specify it with `yaml` file, `ini` file, `json` file, Python `dict` or simply as Python code.
+
+If [rich handler](https://rich.readthedocs.io/en/stable/logging.html) is used, you can only write the configuration as Python code.
+
+# References:
+
+- [Logging for ML Systems](https://madewithml.com/courses/mlops/logging/)
+- [Python Logging Guide – Best Practices and Hands-on Examples](https://coralogix.com/blog/python-logging-best-practices-tips/)

app/proto-docs/docs/virtual_environment.md

@@ -0,0 +1,113 @@
+# Pyenv and Poetry
+
+Pyenv is meant to manage multiple Python versions installed in the computer. It allows you to download different Python versions. You can set a global Python version and a different local Python version for a specific project, so when you enter that directory, the Python version is automatically switched to the one you set.
+
+Poetry is meant to manage virtual environment of a Python project. Poetry does not have a way to download Python specific version needed in a project, so it requires Pyenv to provide one.
+
+## Typical Workflow
+
+You need to replace `<project name>` and `<Python version>` based on your needs. For example, you may set `<project name>` to `my-project` and `<Python version>` to `3.11.3`.
+
+### New Project
+
+1. Decide which Python version should be used for the new project.
+
+2. Ensure you have the Python version needed in the computer. You may check it through `pyenv versions`
+
+    If it is not installed yet, you may run `pyenv install <Python version>` in the terminal.
+
+    !!! note
+
+        You can also check installable Python list by running `pyenv install --list`
+    
+
+3. Create the new project folder by running `poetry new --src <project name>` and change directory to the project folder `cd <project name>`
+
+4. Now set the Python version locally `pyenv local <Python version>`
+
+5. Ensure Python version requirement in `pyproject.toml` satisfies Python version specified in the previous step
+
+6. Set Poetry to create the virtual environment and prefer active Python in the path
+    ```
+    poetry config virtualenvs.in-project true --local 
+    poetry config virtualenvs.prefer-active-python true --local
+    ```
+
+    !!! note
+
+        `in-project` is needed to ensure the virtual environment folder `.venv` is created in the project root folder. This is useful when you want find this folder easily during docker multi-stage build.
+
+        `prefer-active-python` is needed to ensure Poetry uses the Python version specified in the previous step.
+
+        You can also set the configuration globally by replacing `--local` with `--global`. This is useful when you want to use the same configuration for all projects.
+
+7. Create the Python virtual environment by running `poetry install`
+
+8. Activate the virtual environment by running `poetry shell`
+
+9. You can specify third-party dependencies with `poetry add` ([doc here](https://python-poetry.org/docs/cli/#add)) in the terminal or directly edit `pyproject.toml`
+
+### Convert Existing Project
+
+!!! tip
+
+    It is usually unnecessary to convert existing Python virtual environment manager to Poetry if you already have one. However, if you want to convert it, you can follow the steps below.
+
+It follows the same steps above, except in creating project folder since you already existing one. So instead of running `poetry new --src <project name>`, run `poetry init --no-interaction --python <Python version>`.
+
+- References:
+
+    - [Pyenv docs](https://github.com/pyenv/pyenv)
+    - [Poetry docs](https://python-poetry.org/docs/)
+
+# Conda or Mamba
+
+## Typical Workflow
+
+You need to replace `<project name>`, `<Python version>`, and `<environment name>` based on your needs. For example, you may set `<project name>` to `my-project`, `<Python version>` to `3.11.3`, and `<environment name>` to `my-env`.
+
+If `mamba` is installed, replace `conda` command with `mamba` for faster third-party packages installation.
+
+### New Project
+
+1. Create new folder `mkdir <project name> && cd <project name>`
+
+2. Specify the Python virtual environment requirements by creating  `environment.yml` file in project root folder
+
+    ```yaml
+    name: <environment name>
+    channels:
+        - conda-forge
+    dependencies:
+        - python=<Python version>
+        - pandas=2.0
+        - ...
+    ```
+3. Create Python virtual environment and install third party dependencies by running `conda env create -f environment.yml`
+
+4. Activate the environment by running `conda activate <environment name>`
+
+5. You can specify third-party dependencies with `conda install` ([doc here](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html)) in the terminal or directly edit `environment.yml`
+
+- References:
+
+    - [Conda docs](https://docs.conda.io/projects/conda/en/latest/user-guide/index.html)
+
+### Convert Existing Project
+
+!!! tip
+
+    It is usually unnecessary to convert existing Python virtual environment manager to Conda if you already have one. However, if you want to convert it, you can follow the steps below.
+
+It follows the same steps above, except in creating project folder since you already existing one. You may freeze the existing dependencies through `pip freeze > requirements.txt` and create `environment.yml` file with the following content:
+
+```yaml
+name: <environment name>
+channels:
+    - conda-forge
+dependencies:
+    - python=<Python version>
+    - pip
+    - pip:
+        - -r requirements.txt
+```

app/proto-docs/mkdocs.yml

@@ -0,0 +1,46 @@
+site_name: Project Docs
+
+nav:
+  - Home: index.md
+  - Computer Setup: computer_setup.md
+  - Virtual Environment: virtual_environment.md
+  - Code Quality: code_quality.md
+  - Logging: logging.md
+  - Documentation: documentation.md
+
+plugins:
+  - search
+  - mkdocstrings
+  - mkdocs-jupyter:
+      execute: false
+      include_source: true
+
+watch:
+  - src/proto_docs
+
+theme:
+  name: material
+  features:
+    - navigation.top
+  palette:
+    - media: '(prefers-color-scheme: dark)'
+      scheme: slate
+      primary: deep purple
+      accent: purple
+      toggle:
+        icon: material/brightness-7
+        name: Switch to light mode
+    - media: '(prefers-color-scheme: light)'
+      scheme: default
+      primary: blue grey
+      accent: light blue
+      toggle:
+        icon: material/brightness-4
+        name: Switch to dark mode
+
+markdown_extensions: 
+  - admonition
+  - pymdownx.details
+  - pymdownx.superfences
+  - pymdownx.tabbed:
+      alternate_style: true 

app/proto-docs/pyproject.toml

@@ -0,0 +1,39 @@
+[tool.poetry]
+name = "proto-docs"
+version = "0.1.0"
+description = ""
+authors = ["Lee Hong Kai <leehongkai@gmail.com>"]
+readme = "README.md"
+packages = [{ include = "proto_docs", from = "src" }]
+
+[tool.poetry.dependencies]
+python = "~3.11"
+typer = { extras = ["all"], version = "~0.9.0" }
+rich = "~13.3.5"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "~7.3.1"
+coverage = "~7.2.5"
+pytest-cov = "~4.0.0"
+pytest-mock = "~3.10.0"
+jupyter = "~1.0.0"
+black = "~23.3.0"
+ruff = "~0.0.267"
+pyright = "~1.1.308"
+tox = "~4.5.1"
+
+
+[tool.poetry.group.doc.dependencies]
+mkdocs = "~1.4.3"
+mkdocs-material = "~9.1.12"
+mkdocstrings = { extras = ["python"], version = "~0.21.2" }
+mkdocs-jupyter = "~0.24.1"
+
+[tool.poetry.scripts]
+main = "proto_docs.cli:app"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

app/proto-docs/requirements.txt

@@ -0,0 +1,139 @@
+anyio==3.6.2 ; python_version >= "3.11" and python_version < "3.12"
+appnope==0.1.3 ; python_version >= "3.11" and python_version < "3.12" and platform_system == "Darwin" or python_version >= "3.11" and python_version < "3.12" and sys_platform == "darwin"
+argon2-cffi-bindings==21.2.0 ; python_version >= "3.11" and python_version < "3.12"
+argon2-cffi==21.3.0 ; python_version >= "3.11" and python_version < "3.12"
+arrow==1.2.3 ; python_version >= "3.11" and python_version < "3.12"
+asttokens==2.2.1 ; python_version >= "3.11" and python_version < "3.12"
+attrs==23.1.0 ; python_version >= "3.11" and python_version < "3.12"
+backcall==0.2.0 ; python_version >= "3.11" and python_version < "3.12"
+beautifulsoup4==4.12.2 ; python_version >= "3.11" and python_version < "3.12"
+black==23.3.0 ; python_version >= "3.11" and python_version < "3.12"
+bleach==6.0.0 ; python_version >= "3.11" and python_version < "3.12"
+cachetools==5.3.0 ; python_version >= "3.11" and python_version < "3.12"
+certifi==2023.5.7 ; python_version >= "3.11" and python_version < "3.12"
+cffi==1.15.1 ; python_version >= "3.11" and python_version < "3.12"
+chardet==5.1.0 ; python_version >= "3.11" and python_version < "3.12"
+charset-normalizer==3.1.0 ; python_version >= "3.11" and python_version < "3.12"
+click==8.1.3 ; python_version >= "3.11" and python_version < "3.12"
+colorama==0.4.6 ; python_version >= "3.11" and python_version < "3.12"
+comm==0.1.3 ; python_version >= "3.11" and python_version < "3.12"
+coverage==7.2.5 ; python_version >= "3.11" and python_version < "3.12"
+coverage[toml]==7.2.5 ; python_version >= "3.11" and python_version < "3.12"
+debugpy==1.6.7 ; python_version >= "3.11" and python_version < "3.12"
+decorator==5.1.1 ; python_version >= "3.11" and python_version < "3.12"
+defusedxml==0.7.1 ; python_version >= "3.11" and python_version < "3.12"
+distlib==0.3.6 ; python_version >= "3.11" and python_version < "3.12"
+executing==1.2.0 ; python_version >= "3.11" and python_version < "3.12"
+fastjsonschema==2.16.3 ; python_version >= "3.11" and python_version < "3.12"
+filelock==3.12.0 ; python_version >= "3.11" and python_version < "3.12"
+fqdn==1.5.1 ; python_version >= "3.11" and python_version < "3.12"
+ghp-import==2.1.0 ; python_version >= "3.11" and python_version < "3.12"
+griffe==0.27.5 ; python_version >= "3.11" and python_version < "3.12"
+idna==3.4 ; python_version >= "3.11" and python_version < "3.12"
+iniconfig==2.0.0 ; python_version >= "3.11" and python_version < "3.12"
+ipykernel==6.23.1 ; python_version >= "3.11" and python_version < "3.12"
+ipython-genutils==0.2.0 ; python_version >= "3.11" and python_version < "3.12"
+ipython==8.13.2 ; python_version >= "3.11" and python_version < "3.12"
+ipywidgets==8.0.6 ; python_version >= "3.11" and python_version < "3.12"
+isoduration==20.11.0 ; python_version >= "3.11" and python_version < "3.12"
+jedi==0.18.2 ; python_version >= "3.11" and python_version < "3.12"
+jinja2==3.1.2 ; python_version >= "3.11" and python_version < "3.12"
+jsonpointer==2.3 ; python_version >= "3.11" and python_version < "3.12"
+jsonschema==4.17.3 ; python_version >= "3.11" and python_version < "3.12"
+jsonschema[format-nongpl]==4.17.3 ; python_version >= "3.11" and python_version < "3.12"
+jupyter-client==8.2.0 ; python_version >= "3.11" and python_version < "3.12"
+jupyter-console==6.6.3 ; python_version >= "3.11" and python_version < "3.12"
+jupyter-core==5.3.0 ; python_version >= "3.11" and python_version < "3.12"
+jupyter-events==0.6.3 ; python_version >= "3.11" and python_version < "3.12"
+jupyter-server-terminals==0.4.4 ; python_version >= "3.11" and python_version < "3.12"
+jupyter-server==2.5.0 ; python_version >= "3.11" and python_version < "3.12"
+jupyter==1.0.0 ; python_version >= "3.11" and python_version < "3.12"
+jupyterlab-pygments==0.2.2 ; python_version >= "3.11" and python_version < "3.12"
+jupyterlab-widgets==3.0.7 ; python_version >= "3.11" and python_version < "3.12"
+jupytext==1.14.5 ; python_version >= "3.11" and python_version < "3.12"
+markdown-it-py==2.2.0 ; python_version >= "3.11" and python_version < "3.12"
+markdown==3.3.7 ; python_version >= "3.11" and python_version < "3.12"
+markupsafe==2.1.2 ; python_version >= "3.11" and python_version < "3.12"
+matplotlib-inline==0.1.6 ; python_version >= "3.11" and python_version < "3.12"
+mdit-py-plugins==0.3.5 ; python_version >= "3.11" and python_version < "3.12"
+mdurl==0.1.2 ; python_version >= "3.11" and python_version < "3.12"
+mergedeep==1.3.4 ; python_version >= "3.11" and python_version < "3.12"
+mistune==2.0.5 ; python_version >= "3.11" and python_version < "3.12"
+mkdocs-autorefs==0.4.1 ; python_version >= "3.11" and python_version < "3.12"
+mkdocs-jupyter==0.24.1 ; python_version >= "3.11" and python_version < "3.12"
+mkdocs-material-extensions==1.1.1 ; python_version >= "3.11" and python_version < "3.12"
+mkdocs-material==9.1.12 ; python_version >= "3.11" and python_version < "3.12"
+mkdocs==1.4.3 ; python_version >= "3.11" and python_version < "3.12"
+mkdocstrings-python==1.0.0 ; python_version >= "3.11" and python_version < "3.12"
+mkdocstrings==0.21.2 ; python_version >= "3.11" and python_version < "3.12"
+mkdocstrings[python]==0.21.2 ; python_version >= "3.11" and python_version < "3.12"
+mypy-extensions==1.0.0 ; python_version >= "3.11" and python_version < "3.12"
+nbclassic==1.0.0 ; python_version >= "3.11" and python_version < "3.12"
+nbclient==0.7.4 ; python_version >= "3.11" and python_version < "3.12"
+nbconvert==7.4.0 ; python_version >= "3.11" and python_version < "3.12"
+nbformat==5.8.0 ; python_version >= "3.11" and python_version < "3.12"
+nest-asyncio==1.5.6 ; python_version >= "3.11" and python_version < "3.12"
+nodeenv==1.8.0 ; python_version >= "3.11" and python_version < "3.12"
+notebook-shim==0.2.3 ; python_version >= "3.11" and python_version < "3.12"
+notebook==6.5.4 ; python_version >= "3.11" and python_version < "3.12"
+packaging==23.1 ; python_version >= "3.11" and python_version < "3.12"
+pandocfilters==1.5.0 ; python_version >= "3.11" and python_version < "3.12"
+parso==0.8.3 ; python_version >= "3.11" and python_version < "3.12"
+pathspec==0.11.1 ; python_version >= "3.11" and python_version < "3.12"
+pexpect==4.8.0 ; python_version >= "3.11" and python_version < "3.12" and sys_platform != "win32"
+pickleshare==0.7.5 ; python_version >= "3.11" and python_version < "3.12"
+platformdirs==3.5.1 ; python_version >= "3.11" and python_version < "3.12"
+pluggy==1.0.0 ; python_version >= "3.11" and python_version < "3.12"
+prometheus-client==0.16.0 ; python_version >= "3.11" and python_version < "3.12"
+prompt-toolkit==3.0.38 ; python_version >= "3.11" and python_version < "3.12"
+psutil==5.9.5 ; python_version >= "3.11" and python_version < "3.12"
+ptyprocess==0.7.0 ; python_version >= "3.11" and python_version < "3.12" and os_name != "nt" or python_version >= "3.11" and python_version < "3.12" and sys_platform != "win32"
+pure-eval==0.2.2 ; python_version >= "3.11" and python_version < "3.12"
+pycparser==2.21 ; python_version >= "3.11" and python_version < "3.12"
+pygments==2.15.1 ; python_version >= "3.11" and python_version < "3.12"
+pymdown-extensions==10.0.1 ; python_version >= "3.11" and python_version < "3.12"
+pyproject-api==1.5.1 ; python_version >= "3.11" and python_version < "3.12"
+pyright==1.1.308 ; python_version >= "3.11" and python_version < "3.12"
+pyrsistent==0.19.3 ; python_version >= "3.11" and python_version < "3.12"
+pytest-cov==4.0.0 ; python_version >= "3.11" and python_version < "3.12"
+pytest-mock==3.10.0 ; python_version >= "3.11" and python_version < "3.12"
+pytest==7.3.1 ; python_version >= "3.11" and python_version < "3.12"
+python-dateutil==2.8.2 ; python_version >= "3.11" and python_version < "3.12"
+python-json-logger==2.0.7 ; python_version >= "3.11" and python_version < "3.12"
+pywin32==306 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_version >= "3.11" and python_version < "3.12"
+pywinpty==2.0.10 ; python_version >= "3.11" and python_version < "3.12" and os_name == "nt"
+pyyaml-env-tag==0.1 ; python_version >= "3.11" and python_version < "3.12"
+pyyaml==6.0 ; python_version >= "3.11" and python_version < "3.12"
+pyzmq==25.0.2 ; python_version >= "3.11" and python_version < "3.12"
+qtconsole==5.4.3 ; python_version >= "3.11" and python_version < "3.12"
+qtpy==2.3.1 ; python_version >= "3.11" and python_version < "3.12"
+regex==2023.5.5 ; python_version >= "3.11" and python_version < "3.12"
+requests==2.30.0 ; python_version >= "3.11" and python_version < "3.12"
+rfc3339-validator==0.1.4 ; python_version >= "3.11" and python_version < "3.12"
+rfc3986-validator==0.1.1 ; python_version >= "3.11" and python_version < "3.12"
+rich==13.3.5 ; python_version >= "3.11" and python_version < "3.12"
+ruff==0.0.267 ; python_version >= "3.11" and python_version < "3.12"
+send2trash==1.8.2 ; python_version >= "3.11" and python_version < "3.12"
+setuptools==67.7.2 ; python_version >= "3.11" and python_version < "3.12"
+shellingham==1.5.0.post1 ; python_version >= "3.11" and python_version < "3.12"
+six==1.16.0 ; python_version >= "3.11" and python_version < "3.12"
+sniffio==1.3.0 ; python_version >= "3.11" and python_version < "3.12"
+soupsieve==2.4.1 ; python_version >= "3.11" and python_version < "3.12"
+stack-data==0.6.2 ; python_version >= "3.11" and python_version < "3.12"
+terminado==0.17.1 ; python_version >= "3.11" and python_version < "3.12"
+tinycss2==1.2.1 ; python_version >= "3.11" and python_version < "3.12"
+toml==0.10.2 ; python_version >= "3.11" and python_version < "3.12"
+tornado==6.3.2 ; python_version >= "3.11" and python_version < "3.12"
+tox==4.5.1 ; python_version >= "3.11" and python_version < "3.12"
+traitlets==5.9.0 ; python_version >= "3.11" and python_version < "3.12"
+typer[all]==0.9.0 ; python_version >= "3.11" and python_version < "3.12"
+typing-extensions==4.5.0 ; python_version >= "3.11" and python_version < "3.12"
+uri-template==1.2.0 ; python_version >= "3.11" and python_version < "3.12"
+urllib3==2.0.2 ; python_version >= "3.11" and python_version < "3.12"
+virtualenv==20.23.0 ; python_version >= "3.11" and python_version < "3.12"
+watchdog==3.0.0 ; python_version >= "3.11" and python_version < "3.12"
+wcwidth==0.2.6 ; python_version >= "3.11" and python_version < "3.12"
+webcolors==1.13 ; python_version >= "3.11" and python_version < "3.12"
+webencodings==0.5.1 ; python_version >= "3.11" and python_version < "3.12"
+websocket-client==1.5.1 ; python_version >= "3.11" and python_version < "3.12"
+widgetsnbextension==4.0.7 ; python_version >= "3.11" and python_version < "3.12"

app/proto-docs/src/proto_docs/cli.py

@@ -0,0 +1,22 @@
+from typing import Annotated
+
+import typer
+from rich import print
+
+# logger = logging.getLogger(__name__)
+from proto_docs.logger import logger
+
+app = typer.Typer()
+
+
+@app.callback()
+def main():
+    logger.debug("Calling main() function")
+    logger.debug("Exiting main() function")
+
+
+@app.command()
+def commander_1(arg_1: Annotated[int, typer.Option()] = 1):
+    logger.info("Calling commander_1 function")
+    print(f"Another command with {arg_1=}")
+    logger.info("Exiting commander_1 function")

app/proto-docs/src/proto_docs/logger.py

@@ -0,0 +1,44 @@
+import logging
+import logging.config
+import logging.handlers
+from pathlib import Path
+
+from rich.logging import RichHandler
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent.absolute()
+LOGS_DIR = Path(BASE_DIR, "logs")
+LOGS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+# Create handlers
+console_handler = RichHandler(markup=True)
+console_handler.setLevel(logging.DEBUG)
+info_handler = logging.handlers.RotatingFileHandler(
+    filename=Path(LOGS_DIR, "info.log"),
+    maxBytes=10485760,  # 1 MB
+    backupCount=10,
+)
+info_handler.setLevel(logging.INFO)
+error_handler = logging.handlers.RotatingFileHandler(
+    filename=Path(LOGS_DIR, "error.log"),
+    maxBytes=10485760,  # 1 MB
+    backupCount=10,
+)
+error_handler.setLevel(logging.ERROR)
+
+# Create formatters
+minimal_formatter = logging.Formatter(fmt="%(message)s")
+detailed_formatter = logging.Formatter(
+    fmt="%(levelname)s %(asctime)s [%(name)s:%(filename)s:%(funcName)s:%(lineno)d]\n%(message)s\n"
+)
+
+# Hook it all up
+console_handler.setFormatter(fmt=minimal_formatter)
+info_handler.setFormatter(fmt=detailed_formatter)
+error_handler.setFormatter(fmt=detailed_formatter)
+logger.addHandler(hdlr=console_handler)
+logger.addHandler(hdlr=info_handler)
+logger.addHandler(hdlr=error_handler)

doc/README.md

@@ -0,0 +1 @@
+##  save here your general documentation e.g. docs received from customer, architecture, components, mockups, ui sketches

infrastructure/README.md

@@ -0,0 +1 @@
+## save here your infrastructure as code e.g. terraform or azure services

notebooks/README.md

@@ -0,0 +1 @@
+## save here all your jupyter notebooks

pipelines/README.md

@@ -0,0 +1,19 @@
+# Pipelines
+
+## Global Pipeline Templates
+
+To use the global pipeline templates, you simply have to add the repo as a resource and reference the template like this:
+
+```yaml
+## we need to reference the Template Repo here
+resources:
+  repositories:
+    - repository: pipeline-templates
+      type: git
+      name: pipeline-templates
+      ref: refs/tags/v1.0 # Tagged Version of Templates
+# ...
+- template: pipelines/template_build.yml@pipeline-templates
+  parameters: 
+            # ...
+```

pipelines/templates/template_test.yml

@@ -0,0 +1,56 @@
+parameters:
+  - name: projectName
+    type: string
+  - name: packageName
+    type: string
+  - name: workingDirectory
+    type: string
+
+jobs:
+  - job: Build
+    timeoutInMinutes: 120
+    pool:
+      vmImage: "ubuntu-latest"
+    variables:
+      PYTHON_VERSION: 3.9
+      POETRY_VERSION: 1.4.2
+      
+
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{ variables.PYTHON_VERSION }}
+    - checkout: self
+      fetchDepth: 1
+      displayName: "Checkout repo"
+    - script: pip install poetry==${{ variables.POETRY_VERSION }} pytest-azurepipelines
+      displayName: "Install pipeline dependencies"
+    - script: poetry install
+      displayName: "Setup environment using Poetry"
+      workingDirectory: ${{ parameters.workingDirectory }}
+    - script: make lint-check
+      displayName: "Lint check using Ruff"
+      workingDirectory: ${{ parameters.workingDirectory }}
+    - script: make format-check
+      displayName: "Format check using Black"
+      workingDirectory: ${{ parameters.workingDirectory }}
+    - script: make statictypes-check
+      displayName: "Static types check using Pyright"
+      workingDirectory: ${{ parameters.workingDirectory }}
+    - script: |
+        poetry run pytest --junitxml=results.xml  --cov=${{ parameters.packageName }} --cov-report xml
+      displayName: "Execute unit tests using Pytest"
+      workingDirectory: ${{ parameters.workingDirectory }}
+    - task: PublishTestResults@2
+      displayName: "upload Test Results"
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFormat: "JUnit"
+        testResultsFiles: "$(System.DefaultWorkingDirectory)/${{ parameters.workingDirectory }}/results.xml"
+        testRunTitle: "${{ parameters.projectName }} | $(Build.SourceVersionMessage)/$(Build.BuildId)"
+    - task: PublishCodeCoverageResults@1
+      displayName: "Upload code coverage Results"
+      inputs:
+        codeCoverageTool: 'Cobertura'
+        summaryFileLocation: "$(System.DefaultWorkingDirectory)/${{ parameters.workingDirectory }}/coverage.xml"
+        pathToSources: $(System.DefaultWorkingDirectory)/${{ parameters.workingDirectory }}/src/

scripts/README.md

@@ -0,0 +1 @@
+## save all your scripts that do not fit into the the app folder e.g. you generate data only for one time