vlmm_metadata #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily Python Script Execution | ||
| on: | ||
| pull_request: | ||
| schedule: | ||
| # Runs at 00:00 UTC every day | ||
| - cron: '0 0 * * *' | ||
| jobs: | ||
| run-python-script: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.10' # Ensure this matches the Python version you need | ||
| - name: Install dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install requests huggingface-hub | ||
| - name: Execute Python script | ||
| env: | ||
| HF_TOKEN: ${{ secrets.HF_TOKEN }} # Make sure to set this secret in your repository settings | ||
| run: python -c ' | ||
| import os | ||
| import ast | ||
| import json | ||
| import requests | ||
| from huggingface_hub import HfApi | ||
| def extract_models_sub_dict(parsed_code, sub_dict_name): | ||
| class MODELS_SUB_LIST_VISITOR(ast.NodeVisitor): | ||
| def __init__(self): | ||
| self.key = sub_dict_name | ||
| self.value = None | ||
| def visit_Assign(self, node): | ||
| for target in node.targets: | ||
| if isinstance(target, ast.Name) and target.id == self.key: | ||
| self.value = ast.literal_eval(node.value) | ||
| visitor = MODELS_SUB_LIST_VISITOR() | ||
| visitor.visit(parsed_code) | ||
| return visitor.value | ||
| def extract_models_dict(source_code): | ||
| parsed_code = ast.parse(source_code) | ||
| class MODELS_LIST_VISITOR(ast.NodeVisitor): | ||
| def __init__(self): | ||
| self.key = "_MODELS" | ||
| self.value = {} | ||
| def visit_Assign(self, node): | ||
| for target in node.targets: | ||
| if not isinstance(target, ast.Name): | ||
| return | ||
| if target.id == self.key: | ||
| for value in node.value.values: | ||
| dict = extract_models_sub_dict(parsed_code, value.id) | ||
| self.value.update(dict) | ||
| visitor = MODELS_LIST_VISITOR() | ||
| visitor.visit(parsed_code) | ||
| return visitor.value | ||
| # Fetch the content of the file | ||
| url = "https://raw.githubusercontent.com/vllm-project/vllm/refs/heads/main/vllm/model_executor/models/registry.py" | ||
| response = requests.get(url) | ||
| response.raise_for_status() # Raise an exception for bad status codes | ||
| source_code = response.text | ||
| if __name__ == '__main__': | ||
| # extract models dict that consists of sub dicts | ||
| # _MODELS = { | ||
| # **_TEXT_GENERATION_MODELS, | ||
| # **_EMBEDDING_MODELS, | ||
| # **_MULTIMODAL_MODELS, | ||
| # **_SPECULATIVE_DECODING_MODELS, | ||
| # } | ||
| # _TEXT_GENERATION_MODELS = { | ||
| # "AquilaModel": ("llama", "LlamaForCausalLM"), | ||
| # "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2 | ||
| # "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"), | ||
| # ... | ||
| models_dict = extract_models_dict(source_code) | ||
| architectures = [item for tup in models_dict.values() for item in tup] | ||
| architectures_json_str = json.dumps(architectures, indent=4) | ||
| json_bytes = architectures_json_str.encode('utf-8') | ||
| print(architectures_json_str) | ||
| # api = HfApi(token=os.environ["HF_TOKEN"]) | ||
| # api.upload_file( | ||
| # path_or_fileobj=json_bytes, | ||
| # path_in_repo="archtiectures.json", | ||
| # repo_id="mishig/test-vllm", | ||
| # repo_type="dataset", | ||
| # )' | ||
