forked from docs/doc-exports
422 lines
16 KiB
YAML
422 lines
16 KiB
YAML
name: Install Pandoc from Source
|
|
on:
|
|
pull_request:
|
|
types: [opened, reopened, synchronize, edited]
|
|
|
|
jobs:
|
|
install-pandoc:
|
|
runs-on: ubuntu-latest # adjust to your runner label
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Download Pandoc 2.19.2
|
|
run: |
|
|
curl -L -o pandoc.tar.gz \
|
|
https://github.com/jgm/pandoc/releases/download/2.19.2/pandoc-2.19.2-linux-amd64.tar.gz
|
|
|
|
- name: Extract and Install Pandoc
|
|
run: |
|
|
tar xvzf pandoc.tar.gz --strip-components 1 -C /usr/local/
|
|
chmod +x /usr/local/bin/pandoc
|
|
|
|
- name: Verify Installation
|
|
run: pandoc --version
|
|
|
|
install-convertor:
|
|
runs-on: ubuntu-latest
|
|
# outputs:
|
|
# cache-key: ${{ steps.cache-key.outputs.key }}
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.10' # adjust version as needed
|
|
|
|
# - name: Generate cache key
|
|
# id: cache-key
|
|
# run: |
|
|
# echo "key=venv-${{ hashFiles('requirements.txt') }}" >> $GITHUB_OUTPUT
|
|
|
|
# - name: Cache venv
|
|
# uses: actions/cache@v3
|
|
# with:
|
|
# path: ./venv
|
|
# key: ${{ steps.cache-key.outputs.key }}
|
|
|
|
- name: Clone doc-convertor from Gitea
|
|
env:
|
|
GITEA_TOKEN: ${{ secrets.TEST_TOKEN }}
|
|
GITEA_SERVER_URL: ${{ gitea.server_url }}
|
|
run: |
|
|
git clone https://${GITEA_TOKEN}@gitea.eco.tsi-dev.otc-service.com/docs/doc-convertor.git
|
|
|
|
- name: Create virtualenv and install package in editable mode
|
|
run: |
|
|
python -m venv ~/.venv
|
|
source ~/.venv/bin/activate
|
|
pip install --upgrade pip
|
|
pip install -e ./doc-convertor
|
|
|
|
parse-metadata-yaml:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.10' # adjust version as needed
|
|
|
|
- name: Install PyYAML
|
|
run: pip install pyyaml
|
|
|
|
- name: Load YAML and set outputs
|
|
id: metadata
|
|
shell: python
|
|
run: |
|
|
# python3 <<EOF > parsed.env
|
|
import yaml, os
|
|
with open("metadata.yaml") as f:
|
|
data = yaml.safe_load(f)
|
|
print(f"SERVICE_TITLE={data['categories']['security-services'][0]['service_title']}")
|
|
print(f"SERVICE_TYPE={data['categories']['security-services'][0]['service_type']}")
|
|
print(f"SERVICE_CATEGORY={data['categories']['security-services'][0]['service_category']}")
|
|
# EOF
|
|
# cat parsed.env >> $GITHUB_ENV
|
|
|
|
- name: Use config values
|
|
run: |
|
|
echo "SERVICE_TITLE: $SERVICE_TITLE"
|
|
echo "SERVICE_TYPE: $SERVICE_TYPE"
|
|
echo "SERVICE_CATEGORY: $SERVICE_CATEGORY"
|
|
|
|
|
|
generate-docs:
|
|
# needs: install-convertor
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
DOCS_UPDATE_DATA_FILE: metadata.yaml
|
|
PROJECT_SRC_DIR: ./
|
|
TEMPLATE_DIR: ./templates
|
|
|
|
steps:
|
|
- name: Checkout source repo
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Install Python and dependencies
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: 3.10
|
|
|
|
- name: Download Pandoc 2.19.2
|
|
run: |
|
|
curl -L -o pandoc.tar.gz \
|
|
https://github.com/jgm/pandoc/releases/download/2.19.2/pandoc-2.19.2-linux-amd64.tar.gz
|
|
|
|
- name: Extract and Install Pandoc
|
|
run: |
|
|
tar xvzf pandoc.tar.gz --strip-components 1 -C /usr/local/
|
|
chmod +x /usr/local/bin/pandoc
|
|
|
|
- name: Verify Installation
|
|
run: pandoc --version
|
|
|
|
# - name: Restore venv from cache
|
|
# uses: actions/cache@v3
|
|
# with:
|
|
# path: ./venv
|
|
# key: ${{ needs.install-convertor.outputs.cache-key }}
|
|
|
|
- name: Clone doc-convertor from Gitea
|
|
env:
|
|
GITEA_TOKEN: ${{ secrets.TEST_TOKEN }}
|
|
GITEA_SERVER_URL: ${{ gitea.server_url }}
|
|
run: |
|
|
git clone https://${GITEA_TOKEN}@gitea.eco.tsi-dev.otc-service.com/docs/doc-convertor.git
|
|
|
|
- name: Create virtualenv and install package in editable mode
|
|
run: |
|
|
# python -m venv ~/.venv
|
|
# source ~/.venv/bin/activate
|
|
pip install --upgrade pip
|
|
pip install -e ./doc-convertor
|
|
|
|
- name: Install PyYAML (to parse YAML files)
|
|
run: pip install pyyaml
|
|
|
|
- name: Read project docs configuration and extract needed vars
|
|
id: read_docs_config
|
|
run: |
|
|
python3 <<EOF
|
|
import yaml
|
|
import os
|
|
import json
|
|
from pathlib import Path
|
|
|
|
data_file = os.getenv('DOCS_UPDATE_DATA_FILE')
|
|
with open(data_file) as f:
|
|
data = yaml.safe_load(f)
|
|
|
|
# Simulate categories[docs_service_category] defaulting to {}
|
|
# docs_service_category = data.get('categories', '')
|
|
docs_service_category = "application"
|
|
print(docs_service_category)
|
|
categories = data.get('categories', {})
|
|
print(categories)
|
|
|
|
doc_exports_var = categories.get(docs_service_category, {})
|
|
print(doc_exports_var)
|
|
json_str = json.dumps(doc_exports_var, indent=4)
|
|
json_str = "doc_exports_var=" + json_str
|
|
print(json_str)
|
|
with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
|
|
fh.write(json_str)
|
|
|
|
# Write doc_exports_var to output & env file, serialized as JSON for easy later processing
|
|
with open('doc_exports_var.json', 'w') as jf:
|
|
json.dump(doc_exports_var, jf)
|
|
EOF
|
|
sleep 5
|
|
echo "Loaded doc_exports_var and saved to doc_exports_var.json"
|
|
ls -la ./doc_exports_var.json
|
|
cat ./doc_exports_var.json
|
|
|
|
# - name: Get last git changed files
|
|
# id: git_log
|
|
# run: |
|
|
# cd $PROJECT_SRC_DIR
|
|
# ls
|
|
# echo > ../changed_files.txt
|
|
# git branch
|
|
# git status
|
|
# git log -1 --name-only --pretty= > ../changed_files.txt || true
|
|
# cat ../changed_files.txt
|
|
# cat ../doc_exports_var.json
|
|
|
|
|
|
- name: Install jq
|
|
run: sudo apt-get update && sudo apt-get install -y jq
|
|
|
|
- name: Debug event file
|
|
run: |
|
|
echo "Gitea event file path: ${{ gitea.event_path }}"
|
|
export GT_EVENT_PATH="${{ gitea.event_path }}"
|
|
ls -la $GT_EVENT_PATH
|
|
cat $GT_EVENT_PATH
|
|
PR_NUMBER=$(jq -r .pull_request.number "$GT_EVENT_PATH")
|
|
REPO=$(jq -r .repository.full_name "$GT_EVENT_PATH")
|
|
OWNER=$(echo "$REPO" | cut -d'/' -f1)
|
|
REPO_NAME=$(echo "$REPO" | cut -d'/' -f2)
|
|
echo "REPO=$REPO" >> $GITHUB_ENV
|
|
echo "REPO_NAME=$REPO_NAME" >> $GITHUB_ENV
|
|
echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV
|
|
echo "OWNER=$OWNER" >> $GITHUB_ENV
|
|
# exit 1
|
|
|
|
- name: List changed files in PR
|
|
id: changed_files
|
|
run: |
|
|
# PR_NUMBER=$(jq -r .pull_request.number "$GT_EVENT_PATH")
|
|
# REPO=$(jq -r .repository.full_name "$GT_EVENT_PATH")
|
|
|
|
echo "PR #$PR_NUMBER from repo $REPO"
|
|
|
|
# Extract owner and repo name
|
|
# OWNER=$(echo "$REPO" | cut -d'/' -f1)
|
|
# REPO_NAME=$(echo "$REPO" | cut -d'/' -f2)
|
|
echo "Repository $REPO_NAME in organization $OWNER"
|
|
|
|
# Use Gitea API to get changed files
|
|
# Requires an API token with repo read access
|
|
curl -s -H "Authorization: token ${{ secrets.TEST_TOKEN }}" \
|
|
"https://gitea.eco.tsi-dev.otc-service.com/api/v1/repos/$OWNER/$REPO_NAME/pulls/$PR_NUMBER/files" \
|
|
| jq -r '.[].filename' > changed_files.txt
|
|
|
|
echo "Changed files:"
|
|
cat changed_files.txt
|
|
|
|
- name: Generate RST docs
|
|
shell: python
|
|
run: |
|
|
# source ~/.venv/bin/activate
|
|
# python3 <<EOF
|
|
import json
|
|
import os
|
|
import re
|
|
from subprocess import run
|
|
|
|
# Load doc_exports_var from JSON
|
|
with open('doc_exports_var.json') as f:
|
|
doc_exports_var = json.load(f)
|
|
|
|
# Read changed files
|
|
with open('changed_files.txt') as f:
|
|
changed_files = f.read()
|
|
|
|
def file_changed(pattern):
|
|
return re.search(pattern, changed_files) is not None
|
|
|
|
# Loop through docs (assuming structure similar to ansible)
|
|
for doc0 in doc_exports_var:
|
|
# Here doc0 = {service_type:..., repository:..., service_title:..., ...}
|
|
for doc1 in doc0.get('docs', []):
|
|
html_loc = doc1.get('html_location')
|
|
rst_loc = doc1.get('rst_location')
|
|
|
|
# Skip if html location or rst location is not defined
|
|
if not (html_loc and rst_loc):
|
|
continue
|
|
|
|
# Check if changed_files contain relevant paths
|
|
if (file_changed(html_loc) or
|
|
file_changed('metadata.yaml') or
|
|
file_changed('playbooks') or
|
|
file_changed('templates')):
|
|
|
|
# Build variables for convert_doc role
|
|
env_vars = {
|
|
"DOC_LABEL": f"{doc0.get('service_type')}_{doc1.get('type')}",
|
|
"DEST": os.path.expanduser(f"{doc0.get('repository')}/{doc1.get('rst_location')}"),
|
|
"SOURCE": os.path.join(os.getenv('PROJECT_SRC_DIR'), doc1.get('html_location')),
|
|
"DOC_TITLE": f"{doc0.get('service_title')} - {doc1.get('title')}",
|
|
"DOC_SHORT_TITLE": doc1.get('title'),
|
|
"DOC_ENVIRONMENT": doc0.get('service_environment'),
|
|
"DOC_LINK": doc1.get('link'),
|
|
"DOC_TYPE": doc1.get('type'),
|
|
"DOC_SERVICE": doc0.get('service_title'),
|
|
"DOC_SERVICE_CATEGORY": doc0.get('service_category'),
|
|
"DOC_SERVICE_TYPE": doc0.get('service_type'),
|
|
"DOC_REPO_NAME": doc0.get('repository'),
|
|
"DOC_PDF_NAME": doc1.get('pdf_name'),
|
|
}
|
|
print(f"env_vars for doc build: {env_vars}")
|
|
doc_parameters = json.dumps(env_vars, indent=4)
|
|
doc_parameters = "doc_parameters=" + doc_parameters
|
|
with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
|
|
fh.write(json.dumps(doc_parameters, indent=4))
|
|
# Export env vars for convert_doc command
|
|
convert_params = []
|
|
|
|
def add_param(key, flag=None):
|
|
value = env_vars.get(key)
|
|
if value:
|
|
flag = flag or f"--{key.lower().replace('_', '-')}"
|
|
convert_params.append(f"{flag} '{value}'")
|
|
|
|
# Add all optional params
|
|
add_param("DEST", "--dest")
|
|
add_param("DOC_TITLE", "--title")
|
|
add_param("DOC_SHORT_TITLE", "--doc-title")
|
|
add_param("DOC_ENVIRONMENT", "--doc-environment")
|
|
add_param("DOC_LINK", "--doc-link")
|
|
add_param("DOC_TYPE", "--doc-type")
|
|
add_param("DOC_SERVICE", "--service")
|
|
add_param("DOC_SERVICE", "--service-title") # reused intentionally
|
|
add_param("DOC_SERVICE_CATEGORY", "--service-category")
|
|
add_param("DOC_SERVICE_TYPE", "--service-type")
|
|
add_param("DOC_REPO_NAME", "--repo-name")
|
|
add_param("DOC_PDF_NAME", "--pdf-name")
|
|
|
|
# Always set this one (like in Ansible)
|
|
convert_params.append(f"--templates-location {os.getenv('TEMPLATE_DIR')}")
|
|
|
|
# Build final command
|
|
source = env_vars["SOURCE"]
|
|
convert_params.append(source)
|
|
#command = f"otc-convert-doc {' '.join(convert_params)} '{source}'"
|
|
env_str = ' '.join(convert_params)
|
|
print(f"convert_params for doc build: {env_str}")
|
|
|
|
# env_str = " ".join([f'{k}="{v}"' for k, v in env_vars.items() if v])
|
|
|
|
print(f"Generating docs for {env_vars['DOC_LABEL']}")
|
|
|
|
# Convert the respective doc with metadata parameters
|
|
result = run(f"otc-convert-doc {env_str}", shell=True)
|
|
if result.returncode != 0:
|
|
print(f"Error running convert_doc for {env_vars['DOC_LABEL']}")
|
|
exit(result.returncode)
|
|
else:
|
|
print(f"No files changed found in {html_loc}")
|
|
# EOF
|
|
|
|
- name: Upload artifact
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: doc-exports_var
|
|
path: doc_exports_var.json
|
|
|
|
fetch-doc-artifacts:
|
|
needs: generate-docs
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
DOCS_UPDATE_DATA_FILE: metadata.yaml
|
|
PROJECT_SRC_DIR: ./
|
|
TEMPLATE_DIR: ./templates
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v3
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.10"
|
|
|
|
- name: Install dependencies (if needed)
|
|
run: pip install pyyaml
|
|
|
|
- name: Fetch doc-exports_var
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: doc-exports_var
|
|
|
|
- name: Show the file content
|
|
run: cat output.txt
|
|
|
|
- name: Fetch artifacts
|
|
run: |
|
|
python3 <<EOF
|
|
import os
|
|
import tarfile
|
|
import shutil
|
|
|
|
# Load doc_exports_var from JSON
|
|
with open('doc_exports_var.json') as f:
|
|
doc_exports_var = json.load(f)
|
|
# Setup output artifacts directory
|
|
project_root = os.getcwd()
|
|
output_dir = os.path.join(project_root, 'artifact-logs')
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
for doc0 in doc_exports_var:
|
|
for doc1 in doc0['docs']:
|
|
label = f"{doc0['service_type']}_{doc1['type']}"
|
|
rst_path = os.path.expanduser(os.path.join(doc0['repository'], doc1['rst_location']))
|
|
archive_name = f"{label}.tar.gz"
|
|
archive_path = os.path.join(output_dir, archive_name)
|
|
|
|
if os.path.isdir(rst_path):
|
|
print(f"📦 Archiving {rst_path} -> {archive_path}")
|
|
with tarfile.open(archive_path, "w:gz") as tar:
|
|
tar.add(rst_path, arcname=".")
|
|
|
|
# Look for optional patches
|
|
for patch_type in ['diff', 'git']:
|
|
patch_path = os.getenv(f'FETCH_DOC_PATCH_{patch_type.upper()}')
|
|
if patch_path and os.path.exists(patch_path):
|
|
patch_name = f"{label}.{patch_type}.patch"
|
|
print(f"📄 Copying patch: {patch_path} -> {output_dir}/{patch_name}")
|
|
shutil.copy(patch_path, os.path.join(output_dir, patch_name))
|
|
|
|
EOF
|
|
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: doc-artifacts
|
|
path: artifact-logs/
|