HF Datasets

uv add "mirage-ai[hf]"

import os

from mirage import MountMode, Workspace
from mirage.resource.hf_datasets import HfDatasetsConfig, HfDatasetsResource

config = HfDatasetsConfig(
    repo_id=os.environ["HF_DATASET_REPO"],   # "namespace/dataset-name"
    token=os.environ.get("HF_TOKEN"),
    # Optional:
    # endpoint="https://huggingface.co",
    # revision="main",
    # key_prefix="train/",
)
resource = HfDatasetsResource(config)
ws = Workspace({"/ds": resource}, mode=MountMode.READ)

README.md
data/train-00000-of-01000.parquet
data/train-00001-of-01000.parquet

/ds/
  README.md
  data/
    train-00000-of-01000.parquet
    train-00001-of-01000.parquet

import asyncio
import os

from dotenv import load_dotenv

from mirage import MountMode, Workspace
from mirage.resource.hf_datasets import HfDatasetsConfig, HfDatasetsResource

load_dotenv(".env.development")

config = HfDatasetsConfig(
    repo_id=os.environ.get("HF_DATASET_REPO",
                           "AlienKevin/SWE-ZERO-12M-trajectories"),
    token=os.environ.get("HF_TOKEN"),
)
resource = HfDatasetsResource(config)


async def main() -> None:
    ws = Workspace({"/ds": resource}, mode=MountMode.READ)

    r = await ws.execute("ls /ds/")
    print(await r.stdout_str())

    r = await ws.execute("cat /ds/README.md | head -n 20")
    print(await r.stdout_str())

    r = await ws.execute("find /ds/ -name '*.parquet' | head -n 5")
    print(await r.stdout_str())


if __name__ == "__main__":
    asyncio.run(main())