| import json |
| from functools import lru_cache |
| import re |
| import traceback |
| from typing import Optional |
|
|
| import gradio as gr |
| from huggingface_hub import HfApi, hf_hub_download, hf_hub_url |
| from huggingface_hub.utils import HfHubHTTPError |
|
|
|
|
| DEFAULT_REPO_ID = "mlfoundations-cua-dev/human_eval" |
| IMAGE_EXTENSIONS = ( |
| ".jpg", |
| ".jpeg", |
| ".png", |
| ".bmp", |
| ".gif", |
| ".webp", |
| ".tif", |
| ".tiff", |
| ) |
| INIT_SCREENSHOT_NAMES = {"intial_screenshot", "initial_screenshot"} |
| STEP_FILENAME_PATTERN = re.compile(r"^step_(\d+)(?:\.[^.]+)?$", re.IGNORECASE) |
| TRAJECTORY_FILENAME = "traj.jsonl" |
|
|
|
|
| api = HfApi() |
|
|
|
|
| @lru_cache(maxsize=16) |
| def _list_repo_files(repo_id: str) -> list[str]: |
| """Return all file paths contained in a Hugging Face dataset repository.""" |
| return api.list_repo_files(repo_id=repo_id, repo_type="dataset") |
|
|
|
|
| def _extract_top_level(repo_id: str) -> tuple[list[str], list[str]]: |
| """Split top-level folders and files for the given repository.""" |
| files = _list_repo_files(repo_id) |
| top_level_dirs = sorted({path.split("/", 1)[0] for path in files if "/" in path}) |
| top_level_files = sorted(path for path in files if "/" not in path) |
| return top_level_dirs, top_level_files |
|
|
|
|
| def _get_subdirectories(repo_id: str, directory: str) -> list[str]: |
| """Return the direct subdirectories of the given directory.""" |
| if not directory: |
| return [] |
|
|
| files = [path for path in _list_repo_files(repo_id) if path.startswith(f"{directory}/")] |
| relative_paths = [path[len(directory) + 1 :] for path in files] |
| |
| |
| child_dirs = sorted({rel.split("/", 1)[0] for rel in relative_paths if "/" in rel}) |
| |
| return child_dirs |
|
|
|
|
| def _build_path(*parts) -> str: |
| """Join path parts while skipping empty values.""" |
| return "/".join(part for part in parts if part) |
|
|
|
|
| def _image_sort_key(path: str): |
| filename = path.rsplit("/", 1)[-1] |
| lower_name = filename.lower() |
|
|
| if any(lower_name.startswith(name) for name in INIT_SCREENSHOT_NAMES): |
| return (0, 0) |
|
|
| match = STEP_FILENAME_PATTERN.match(lower_name) |
| if match: |
| return (1, int(match.group(1))) |
|
|
| return (2, lower_name) |
|
|
|
|
| def _load_traj_entries(repo_id: str, directory: str) -> list: |
| """Load trajectory annotations from traj.jsonl within the given directory.""" |
| if not directory: |
| return [] |
|
|
| traj_path = _build_path(directory, TRAJECTORY_FILENAME) |
| repo_files = _list_repo_files(repo_id) |
|
|
| if traj_path not in repo_files: |
| return [] |
|
|
| local_path = hf_hub_download( |
| repo_id=repo_id, |
| filename=traj_path, |
| repo_type="dataset", |
| ) |
|
|
| entries: list = [] |
| with open(local_path, "r", encoding="utf-8") as file: |
| for raw_line in file: |
| stripped = raw_line.strip() |
| if not stripped: |
| continue |
|
|
| parsed = json.loads(stripped) |
| if isinstance(parsed, list): |
| entries.extend(parsed) |
| else: |
| entries.append(parsed) |
|
|
| return entries |
|
|
|
|
| def _load_instruction_file(repo_id: str, filepath: str) -> Optional[str]: |
| """Download a metadata JSON file and extract its instruction field.""" |
| local_path = hf_hub_download( |
| repo_id=repo_id, |
| filename=filepath, |
| repo_type="dataset", |
| ) |
|
|
| with open(local_path, "r", encoding="utf-8") as file: |
| data = json.load(file) |
|
|
| instruction: Optional[str] = None |
| if isinstance(data, dict): |
| instruction = data.get("instruction") |
| elif isinstance(data, list): |
| for entry in data: |
| if isinstance(entry, dict) and "instruction" in entry: |
| instruction = entry["instruction"] |
| break |
|
|
| if instruction is None: |
| return None |
|
|
| if isinstance(instruction, str): |
| return instruction.strip() |
|
|
| return str(instruction) |
|
|
|
|
| def _format_annotation(index: int, annotation) -> str: |
| prefix = f"Step {index + 1}" |
| if isinstance(annotation, str): |
| content = annotation.strip() |
| else: |
| try: |
| content = json.dumps(annotation, ensure_ascii=False) |
| except TypeError: |
| content = str(annotation) |
| return f"{prefix}: {content}" if content else prefix |
|
|
|
|
| def _prepare_gallery_items( |
| repo_id: str, directory: Optional[str] |
| ) -> tuple[list[tuple[str, str]], list[str], str]: |
| """Prepare gallery items, status messages, and instruction text for a directory.""" |
| if not directory: |
| return ( |
| [], |
| ["Select a fifth-level folder to view screenshots."], |
| "ℹ️ Instruction will appear once a fifth-level folder is selected.", |
| ) |
|
|
| prefix = f"{directory}/" |
| files = [path for path in _list_repo_files(repo_id) if path.startswith(prefix)] |
|
|
| image_files = [ |
| path |
| for path in files |
| if "/" not in path[len(prefix) :] |
| and path.lower().endswith(IMAGE_EXTENSIONS) |
| ] |
|
|
| sorted_files = sorted(image_files, key=_image_sort_key) |
| image_urls = [ |
| hf_hub_url(repo_id=repo_id, filename=path, repo_type="dataset") |
| for path in sorted_files |
| ] |
|
|
| status_lines: list[str] = [ |
| f"🖼️ Images: {len(image_urls)}", |
| ] |
|
|
| annotations: list = [] |
| try: |
| annotations = _load_traj_entries(repo_id, directory) |
| except json.JSONDecodeError as error: |
| status_lines.append(f"⚠️ Failed to parse `{TRAJECTORY_FILENAME}`: {error}") |
| except Exception as error: |
| status_lines.append(f"⚠️ Error loading `{TRAJECTORY_FILENAME}`: {error}") |
|
|
| status_lines.append(f"📝 Annotations: {len(annotations)}") |
|
|
| if not image_urls: |
| status_lines.append("⚠️ No images found in this folder.") |
|
|
| if image_urls and not annotations: |
| status_lines.append( |
| f"⚠️ `{TRAJECTORY_FILENAME}` missing or empty; no annotations to display." |
| ) |
|
|
| if annotations and len(annotations) != len(image_urls): |
| status_lines.append( |
| "⚠️ Mismatch between images and annotations; displaying available pairs." |
| ) |
|
|
| folder_name = directory.rsplit("/", 1)[-1] |
| metadata_files = [ |
| path |
| for path in files |
| if "/" not in path[len(prefix) :] |
| and path.lower().endswith(".json") |
| and not path.lower().endswith(".jsonl") |
| ] |
|
|
| instruction_markdown = "" |
| if metadata_files: |
| metadata_files.sort( |
| key=lambda path: ( |
| 0 |
| if path.rsplit("/", 1)[-1].rsplit(".", 1)[0] == folder_name |
| else 1, |
| path.rsplit("/", 1)[-1].lower(), |
| ) |
| ) |
| metadata_path = metadata_files[0] |
| metadata_filename = metadata_path.rsplit("/", 1)[-1] |
| try: |
| instruction_value = _load_instruction_file(repo_id, metadata_path) |
| if instruction_value: |
| instruction_markdown = f"**Instruction:** {instruction_value}" |
| status_lines.append("📋 Instruction: loaded") |
| else: |
| instruction_markdown = ( |
| f"⚠️ Instruction missing in `{metadata_filename}`." |
| ) |
| status_lines.append("⚠️ Instruction: missing in metadata.") |
| except json.JSONDecodeError as error: |
| status_lines.append( |
| f"⚠️ Failed to parse `{metadata_filename}`: {error}" |
| ) |
| instruction_markdown = ( |
| f"⚠️ Unable to parse instruction from `{metadata_filename}`." |
| ) |
| except Exception as error: |
| status_lines.append( |
| f"⚠️ Error loading `{metadata_filename}`: {error}" |
| ) |
| instruction_markdown = ( |
| f"⚠️ Unable to load instruction from `{metadata_filename}`." |
| ) |
| else: |
| instruction_markdown = "⚠️ Instruction file not found in this folder." |
| status_lines.append("⚠️ Instruction: file not found.") |
|
|
| gallery_items: list[tuple[str, str]] = [] |
| for idx, url in enumerate(image_urls): |
| caption = "No annotation available" |
| if idx < len(annotations): |
| caption = _format_annotation(idx, annotations[idx]) |
| gallery_items.append((url, caption)) |
|
|
| return gallery_items, status_lines, instruction_markdown |
|
|
|
|
| def _dropdown_update( |
| *, |
| choices: list[str], |
| value: Optional[str], |
| label: str, |
| filled_info: str, |
| empty_info: str, |
| ): |
| has_choices = bool(choices) |
| return gr.update( |
| choices=choices, |
| value=value if has_choices else None, |
| interactive=has_choices, |
| label=label, |
| info=filled_info if has_choices else empty_info, |
| ) |
|
|
|
|
| def refresh_repo(repo_id: str): |
| try: |
| top_dirs, top_files = _extract_top_level(repo_id) |
| except HfHubHTTPError as error: |
| print(f"[refresh_repo] Hub HTTP error for {repo_id}: {error}", flush=True) |
| print(traceback.format_exc(), flush=True) |
| return ( |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(value=""), |
| gr.update(value=[]), |
| gr.update(value=""), |
| gr.update(value=f"❌ Unable to load repo `{repo_id}`: {error}"), |
| ) |
| except Exception as error: |
| print(f"[refresh_repo] Unexpected error for {repo_id}: {error}", flush=True) |
| print(traceback.format_exc(), flush=True) |
| return ( |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(choices=[], value=None, interactive=False), |
| gr.update(value=""), |
| gr.update(value=[]), |
| gr.update(value=""), |
| gr.update(value=f"❌ Unexpected error loading `{repo_id}`: {error}"), |
| ) |
|
|
| status_lines = [ |
| f"✅ Loaded `{repo_id}`", |
| f"• Top-level folders: {len(top_dirs)}", |
| ] |
|
|
| if top_files: |
| status_lines.append(f"• Loose files at root: {len(top_files)}") |
| if not top_dirs: |
| status_lines.append("• No sub-folders found at root.") |
|
|
| top_value = top_dirs[0] if top_dirs else None |
|
|
| second_dirs = _get_subdirectories(repo_id, top_value) if top_value else [] |
| second_value = second_dirs[0] if second_dirs else None |
|
|
| third_dirs = ( |
| _get_subdirectories(repo_id, _build_path(top_value, second_value)) |
| if second_value |
| else [] |
| ) |
| third_value = third_dirs[0] if third_dirs else None |
|
|
| fourth_dirs = ( |
| _get_subdirectories(repo_id, _build_path(top_value, second_value, third_value)) |
| if third_value |
| else [] |
| ) |
| fourth_value = fourth_dirs[0] if fourth_dirs else None |
|
|
| fifth_dirs = ( |
| _get_subdirectories( |
| repo_id, _build_path(top_value, second_value, third_value, fourth_value) |
| ) |
| if fourth_value |
| else [] |
| ) |
| fifth_value = fifth_dirs[0] if fifth_dirs else None |
|
|
| target_directory = ( |
| _build_path( |
| top_value, |
| second_value, |
| third_value, |
| fourth_value, |
| fifth_value, |
| ) |
| if fifth_value |
| else None |
| ) |
| gallery_items, gallery_status, instruction_markdown = _prepare_gallery_items( |
| repo_id, target_directory |
| ) |
|
|
| first_dropdown_update = _dropdown_update( |
| choices=top_dirs, |
| value=top_value, |
| label="Top-level folders", |
| filled_info="Choose a folder to explore", |
| empty_info="No folders found at the repository root", |
| ) |
|
|
| second_dropdown_update = _dropdown_update( |
| choices=second_dirs, |
| value=second_value, |
| label="Second-level folders", |
| filled_info="Choose a second-level folder", |
| empty_info="No subdirectories under the selected folder", |
| ) |
|
|
| third_dropdown_update = _dropdown_update( |
| choices=third_dirs, |
| value=third_value, |
| label="Third-level folders", |
| filled_info="Choose a third-level folder", |
| empty_info="No third-level folders under the selection", |
| ) |
|
|
| fourth_dropdown_update = _dropdown_update( |
| choices=fourth_dirs, |
| value=fourth_value, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="No fourth-level folders under the selection", |
| ) |
|
|
| fifth_dropdown_update = _dropdown_update( |
| choices=fifth_dirs, |
| value=fifth_value, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="No fifth-level folders under the selection", |
| ) |
|
|
| instruction_update = gr.update(value=instruction_markdown) |
| gallery_update = gr.update(value=gallery_items) |
| annotation_update = gr.update(value="\n".join(gallery_status)) |
| status_update = gr.update(value="\n".join(status_lines)) |
|
|
| return ( |
| first_dropdown_update, |
| second_dropdown_update, |
| third_dropdown_update, |
| fourth_dropdown_update, |
| fifth_dropdown_update, |
| instruction_update, |
| gallery_update, |
| annotation_update, |
| status_update, |
| ) |
|
|
|
|
| def update_second_dropdown(repo_id: str, top_level_dir: str): |
| """Update downstream dropdowns when the top-level selection changes.""" |
| try: |
| if not top_level_dir: |
| empty_second = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Second-level folders", |
| filled_info="Choose a second-level folder", |
| empty_info="Select a top-level folder first", |
| ) |
| empty_third = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Third-level folders", |
| filled_info="Choose a third-level folder", |
| empty_info="Select a higher-level folder first", |
| ) |
| empty_fourth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="Select a higher-level folder first", |
| ) |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Select a higher-level folder first", |
| ) |
| instruction_update = gr.update( |
| value="ℹ️ Select a top-level folder to display its instruction." |
| ) |
| gallery_update = gr.update(value=[]) |
| annotation_update = gr.update( |
| value="Select a top-level folder to load screenshots and annotations." |
| ) |
| return ( |
| empty_second, |
| empty_third, |
| empty_fourth, |
| empty_fifth, |
| instruction_update, |
| gallery_update, |
| annotation_update, |
| ) |
|
|
| second_dirs = _get_subdirectories(repo_id, top_level_dir) |
| second_value = second_dirs[0] if second_dirs else None |
|
|
| third_dirs = ( |
| _get_subdirectories(repo_id, _build_path(top_level_dir, second_value)) |
| if second_value |
| else [] |
| ) |
| third_value = third_dirs[0] if third_dirs else None |
|
|
| fourth_dirs = ( |
| _get_subdirectories( |
| repo_id, _build_path(top_level_dir, second_value, third_value) |
| ) |
| if third_value |
| else [] |
| ) |
| fourth_value = fourth_dirs[0] if fourth_dirs else None |
|
|
| fifth_dirs = ( |
| _get_subdirectories( |
| repo_id, |
| _build_path(top_level_dir, second_value, third_value, fourth_value), |
| ) |
| if fourth_value |
| else [] |
| ) |
| fifth_value = fifth_dirs[0] if fifth_dirs else None |
|
|
| target_directory = ( |
| _build_path( |
| top_level_dir, |
| second_value, |
| third_value, |
| fourth_value, |
| fifth_value, |
| ) |
| if fifth_value |
| else None |
| ) |
| ( |
| gallery_items, |
| gallery_status, |
| instruction_markdown, |
| ) = _prepare_gallery_items(repo_id, target_directory) |
|
|
| return ( |
| _dropdown_update( |
| choices=second_dirs, |
| value=second_value, |
| label="Second-level folders", |
| filled_info="Choose a second-level folder", |
| empty_info="No subdirectories under the selected folder", |
| ), |
| _dropdown_update( |
| choices=third_dirs, |
| value=third_value, |
| label="Third-level folders", |
| filled_info="Choose a third-level folder", |
| empty_info="No third-level folders under the selection", |
| ), |
| _dropdown_update( |
| choices=fourth_dirs, |
| value=fourth_value, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="No fourth-level folders under the selection", |
| ), |
| _dropdown_update( |
| choices=fifth_dirs, |
| value=fifth_value, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="No fifth-level folders under the selection", |
| ), |
| gr.update(value=instruction_markdown), |
| gr.update(value=gallery_items), |
| gr.update(value="\n".join(gallery_status)), |
| ) |
| except Exception as error: |
| print(f"[update_second_dropdown] Error for {repo_id}/{top_level_dir}: {error}", flush=True) |
| print(traceback.format_exc(), flush=True) |
| empty_second = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Second-level folders", |
| filled_info="Choose a second-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| empty_third = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Third-level folders", |
| filled_info="Choose a third-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| empty_fourth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| return ( |
| empty_second, |
| empty_third, |
| empty_fourth, |
| empty_fifth, |
| gr.update(value="Unable to load instruction."), |
| gr.update(value=[]), |
| gr.update(value="Unable to load screenshots or annotations."), |
| ) |
|
|
|
|
| def update_third_dropdown(repo_id: str, top_level_dir: str, second_level_dir: str): |
| """Update the third and fourth dropdowns when the second-level changes.""" |
| try: |
| if not top_level_dir or not second_level_dir: |
| empty_third = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Third-level folders", |
| filled_info="Choose a third-level folder", |
| empty_info="Select higher-level folders first", |
| ) |
| empty_fourth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="Select higher-level folders first", |
| ) |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Select higher-level folders first", |
| ) |
| instruction_update = gr.update( |
| value="ℹ️ Select higher-level folders to display the instruction." |
| ) |
| gallery_update = gr.update(value=[]) |
| annotation_update = gr.update( |
| value="Select higher-level folders to load screenshots and annotations." |
| ) |
| return ( |
| empty_third, |
| empty_fourth, |
| empty_fifth, |
| instruction_update, |
| gallery_update, |
| annotation_update, |
| ) |
|
|
| third_dirs = _get_subdirectories( |
| repo_id, _build_path(top_level_dir, second_level_dir) |
| ) |
| third_value = third_dirs[0] if third_dirs else None |
|
|
| fourth_dirs = ( |
| _get_subdirectories( |
| repo_id, _build_path(top_level_dir, second_level_dir, third_value) |
| ) |
| if third_value |
| else [] |
| ) |
| fourth_value = fourth_dirs[0] if fourth_dirs else None |
|
|
| fifth_dirs = ( |
| _get_subdirectories( |
| repo_id, |
| _build_path(top_level_dir, second_level_dir, third_value, fourth_value), |
| ) |
| if fourth_value |
| else [] |
| ) |
| fifth_value = fifth_dirs[0] if fifth_dirs else None |
|
|
| target_directory = ( |
| _build_path( |
| top_level_dir, |
| second_level_dir, |
| third_value, |
| fourth_value, |
| fifth_value, |
| ) |
| if fifth_value |
| else None |
| ) |
| ( |
| gallery_items, |
| gallery_status, |
| instruction_markdown, |
| ) = _prepare_gallery_items(repo_id, target_directory) |
|
|
| return ( |
| _dropdown_update( |
| choices=third_dirs, |
| value=third_value, |
| label="Third-level folders", |
| filled_info="Choose a third-level folder", |
| empty_info="No third-level folders under the selection", |
| ), |
| _dropdown_update( |
| choices=fourth_dirs, |
| value=fourth_value, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="No fourth-level folders under the selection", |
| ), |
| _dropdown_update( |
| choices=fifth_dirs, |
| value=fifth_value, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="No fifth-level folders under the selection", |
| ), |
| gr.update(value=instruction_markdown), |
| gr.update(value=gallery_items), |
| gr.update(value="\n".join(gallery_status)), |
| ) |
| except Exception as error: |
| print( |
| f"[update_third_dropdown] Error for {repo_id}/{top_level_dir}/{second_level_dir}: {error}", |
| flush=True, |
| ) |
| print(traceback.format_exc(), flush=True) |
| empty_third = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Third-level folders", |
| filled_info="Choose a third-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| empty_fourth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| return ( |
| empty_third, |
| empty_fourth, |
| empty_fifth, |
| gr.update(value="Unable to load instruction."), |
| gr.update(value=[]), |
| gr.update(value="Unable to load screenshots or annotations."), |
| ) |
|
|
|
|
| def update_fourth_dropdown( |
| repo_id: str, |
| top_level_dir: str, |
| second_level_dir: str, |
| third_level_dir: str, |
| ): |
| """Update the fourth and fifth dropdowns when the third-level changes.""" |
| try: |
| if not top_level_dir or not second_level_dir or not third_level_dir: |
| empty_fourth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="Select higher-level folders first", |
| ) |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Select higher-level folders first", |
| ) |
| instruction_update = gr.update( |
| value="ℹ️ Select higher-level folders to display the instruction." |
| ) |
| gallery_update = gr.update(value=[]) |
| annotation_update = gr.update( |
| value="Select higher-level folders to load screenshots and annotations." |
| ) |
| return ( |
| empty_fourth, |
| empty_fifth, |
| instruction_update, |
| gallery_update, |
| annotation_update, |
| ) |
|
|
| fourth_dirs = _get_subdirectories( |
| repo_id, |
| _build_path(top_level_dir, second_level_dir, third_level_dir), |
| ) |
| fourth_value = fourth_dirs[0] if fourth_dirs else None |
|
|
| fifth_dirs = ( |
| _get_subdirectories( |
| repo_id, |
| _build_path( |
| top_level_dir, second_level_dir, third_level_dir, fourth_value |
| ), |
| ) |
| if fourth_value |
| else [] |
| ) |
| fifth_value = fifth_dirs[0] if fifth_dirs else None |
|
|
| target_directory = ( |
| _build_path( |
| top_level_dir, |
| second_level_dir, |
| third_level_dir, |
| fourth_value, |
| fifth_value, |
| ) |
| if fifth_value |
| else None |
| ) |
| ( |
| gallery_items, |
| gallery_status, |
| instruction_markdown, |
| ) = _prepare_gallery_items(repo_id, target_directory) |
|
|
| return ( |
| _dropdown_update( |
| choices=fourth_dirs, |
| value=fourth_value, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="No fourth-level folders under the selection", |
| ), |
| _dropdown_update( |
| choices=fifth_dirs, |
| value=fifth_value, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="No fifth-level folders under the selection", |
| ), |
| gr.update(value=instruction_markdown), |
| gr.update(value=gallery_items), |
| gr.update(value="\n".join(gallery_status)), |
| ) |
| except Exception as error: |
| print( |
| "[update_fourth_dropdown] Error for " |
| f"{repo_id}/{top_level_dir}/{second_level_dir}/{third_level_dir}: {error}", |
| flush=True, |
| ) |
| print(traceback.format_exc(), flush=True) |
| empty_fourth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fourth-level folders", |
| filled_info="Choose a fourth-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| return ( |
| empty_fourth, |
| empty_fifth, |
| gr.update(value="Unable to load instruction."), |
| gr.update(value=[]), |
| gr.update(value="Unable to load screenshots or annotations."), |
| ) |
|
|
|
|
| def update_fifth_dropdown( |
| repo_id: str, |
| top_level_dir: str, |
| second_level_dir: str, |
| third_level_dir: str, |
| fourth_level_dir: str, |
| ): |
| """Update the fifth dropdown and gallery when the fourth-level changes.""" |
| try: |
| if not all( |
| [top_level_dir, second_level_dir, third_level_dir, fourth_level_dir] |
| ): |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Select higher-level folders first", |
| ) |
| instruction_update = gr.update( |
| value="ℹ️ Select higher-level folders to display the instruction." |
| ) |
| gallery_update = gr.update(value=[]) |
| annotation_update = gr.update( |
| value="Select higher-level folders to load screenshots and annotations." |
| ) |
| return empty_fifth, instruction_update, gallery_update, annotation_update |
|
|
| fifth_dirs = _get_subdirectories( |
| repo_id, |
| _build_path( |
| top_level_dir, |
| second_level_dir, |
| third_level_dir, |
| fourth_level_dir, |
| ), |
| ) |
| fifth_value = fifth_dirs[0] if fifth_dirs else None |
|
|
| target_directory = ( |
| _build_path( |
| top_level_dir, |
| second_level_dir, |
| third_level_dir, |
| fourth_level_dir, |
| fifth_value, |
| ) |
| if fifth_value |
| else None |
| ) |
| ( |
| gallery_items, |
| gallery_status, |
| instruction_markdown, |
| ) = _prepare_gallery_items(repo_id, target_directory) |
|
|
| return ( |
| _dropdown_update( |
| choices=fifth_dirs, |
| value=fifth_value, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="No fifth-level folders under the selection", |
| ), |
| gr.update(value=instruction_markdown), |
| gr.update(value=gallery_items), |
| gr.update(value="\n".join(gallery_status)), |
| ) |
| except Exception as error: |
| print( |
| "[update_fifth_dropdown] Error for " |
| f"{repo_id}/{top_level_dir}/{second_level_dir}/{third_level_dir}/{fourth_level_dir}: {error}", |
| flush=True, |
| ) |
| print(traceback.format_exc(), flush=True) |
| empty_fifth = _dropdown_update( |
| choices=[], |
| value=None, |
| label="Fifth-level folders", |
| filled_info="Choose a fifth-level folder", |
| empty_info="Unable to load subdirectories", |
| ) |
| return ( |
| empty_fifth, |
| gr.update(value="Unable to load instruction."), |
| gr.update(value=[]), |
| gr.update(value="Unable to load screenshots or annotations."), |
| ) |
|
|
|
|
| def update_gallery( |
| repo_id: str, |
| top_level_dir: str, |
| second_level_dir: str, |
| third_level_dir: str, |
| fourth_level_dir: str, |
| fifth_level_dir: str, |
| ): |
| """Update the image gallery when the fifth-level selection changes.""" |
| try: |
| directory = ( |
| _build_path( |
| top_level_dir, |
| second_level_dir, |
| third_level_dir, |
| fourth_level_dir, |
| fifth_level_dir, |
| ) |
| if all( |
| [ |
| top_level_dir, |
| second_level_dir, |
| third_level_dir, |
| fourth_level_dir, |
| fifth_level_dir, |
| ] |
| ) |
| else None |
| ) |
| ( |
| gallery_items, |
| gallery_status, |
| instruction_markdown, |
| ) = _prepare_gallery_items(repo_id, directory) |
| return ( |
| gr.update(value=instruction_markdown), |
| gr.update(value=gallery_items), |
| gr.update(value="\n".join(gallery_status)), |
| ) |
| except Exception as error: |
| print( |
| "[update_gallery] Error for " |
| f"{repo_id}/{top_level_dir}/{second_level_dir}/{third_level_dir}/{fourth_level_dir}/{fifth_level_dir}: {error}", |
| flush=True, |
| ) |
| print(traceback.format_exc(), flush=True) |
| return ( |
| gr.update(value="Unable to load instruction."), |
| gr.update(value=[]), |
| gr.update(value="Unable to load screenshots or annotations."), |
| ) |
|
|
|
|
| with gr.Blocks(title="HF Dataset Explorer") as demo: |
| gr.Markdown( |
| """# Hugging Face Dataset Explorer |
| |
| Provide a dataset repository ID (e.g. `org/dataset`) to list its top-level folders.""" |
| ) |
|
|
| with gr.Row(): |
| repo_id_input = gr.Textbox( |
| value=DEFAULT_REPO_ID, |
| label="Dataset repo ID", |
| placeholder="owner/dataset", |
| info="Any public dataset on the Hugging Face Hub" |
| ) |
| reload_button = gr.Button("Load repo", variant="primary") |
|
|
| status_display = gr.Markdown() |
| folder_dropdown = gr.Dropdown(label="Top-level folders", interactive=False) |
| second_level_dropdown = gr.Dropdown(label="Second-level folders", interactive=False) |
| third_level_dropdown = gr.Dropdown(label="Third-level folders", interactive=False) |
| fourth_level_dropdown = gr.Dropdown(label="Fourth-level folders", interactive=False) |
| fifth_level_dropdown = gr.Dropdown(label="Fifth-level folders", interactive=False) |
| instruction_display = gr.Markdown() |
| image_gallery = gr.Gallery(label="Images", columns=4) |
| annotation_status = gr.Markdown() |
|
|
| reload_button.click( |
| refresh_repo, |
| inputs=repo_id_input, |
| outputs=[ |
| folder_dropdown, |
| second_level_dropdown, |
| third_level_dropdown, |
| fourth_level_dropdown, |
| fifth_level_dropdown, |
| instruction_display, |
| image_gallery, |
| annotation_status, |
| status_display, |
| ], |
| ) |
|
|
| folder_dropdown.change( |
| update_second_dropdown, |
| inputs=[repo_id_input, folder_dropdown], |
| outputs=[ |
| second_level_dropdown, |
| third_level_dropdown, |
| fourth_level_dropdown, |
| fifth_level_dropdown, |
| instruction_display, |
| image_gallery, |
| annotation_status, |
| ], |
| ) |
|
|
| second_level_dropdown.change( |
| update_third_dropdown, |
| inputs=[repo_id_input, folder_dropdown, second_level_dropdown], |
| outputs=[ |
| third_level_dropdown, |
| fourth_level_dropdown, |
| fifth_level_dropdown, |
| instruction_display, |
| image_gallery, |
| annotation_status, |
| ], |
| ) |
|
|
| third_level_dropdown.change( |
| update_fourth_dropdown, |
| inputs=[ |
| repo_id_input, |
| folder_dropdown, |
| second_level_dropdown, |
| third_level_dropdown, |
| ], |
| outputs=[ |
| fourth_level_dropdown, |
| fifth_level_dropdown, |
| instruction_display, |
| image_gallery, |
| annotation_status, |
| ], |
| ) |
|
|
| fourth_level_dropdown.change( |
| update_fifth_dropdown, |
| inputs=[ |
| repo_id_input, |
| folder_dropdown, |
| second_level_dropdown, |
| third_level_dropdown, |
| fourth_level_dropdown, |
| ], |
| outputs=[ |
| fifth_level_dropdown, |
| instruction_display, |
| image_gallery, |
| annotation_status, |
| ], |
| ) |
|
|
| fifth_level_dropdown.change( |
| update_gallery, |
| inputs=[ |
| repo_id_input, |
| folder_dropdown, |
| second_level_dropdown, |
| third_level_dropdown, |
| fourth_level_dropdown, |
| fifth_level_dropdown, |
| ], |
| outputs=[instruction_display, image_gallery, annotation_status], |
| ) |
|
|
| demo.load( |
| refresh_repo, |
| inputs=repo_id_input, |
| outputs=[ |
| folder_dropdown, |
| second_level_dropdown, |
| third_level_dropdown, |
| fourth_level_dropdown, |
| fifth_level_dropdown, |
| instruction_display, |
| image_gallery, |
| annotation_status, |
| status_display, |
| ], |
| ) |
|
|
| if __name__ == "__main__": |
| demo.launch() |
