{
  "revisions": {
    "mlx": "bd63d3803008edda03ce74080955d9879e2ef1eb",
    "mlx-lm": "736ab0d8c1d0edc79c3d798044fe6904e2fa23be",
    "llama.cpp": "a91eab67d8389d979d29fdf6633b2edd0718eb4b",
    "model": "7ae557604adf67be50417f59c2c2f167def9a775"
  },
  "source_downloads": {
    "model/LICENSE": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/LICENSE",
      "bytes": 11343,
      "sha256": "832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e",
      "seconds": 0.28911237500142306
    },
    "mlx-lm": {
      "url": "https://codeload.github.com/ml-explore/mlx-lm/tar.gz/736ab0d8c1d0edc79c3d798044fe6904e2fa23be",
      "bytes": 483460,
      "sha256": "c20cc3fc5fa1814f8947befa04447680fd1b9c7df4601174bc569cc60d650f58",
      "seconds": 0.48823825002182275
    },
    "model/config.json": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/config.json",
      "bytes": 659,
      "sha256": "18e18afcaccafade98daf13a54092927904649e1dd4eba8299ab717d5d94ff45",
      "seconds": 0.249934374995064
    },
    "mlx": {
      "url": "https://codeload.github.com/ml-explore/mlx/tar.gz/bd63d3803008edda03ce74080955d9879e2ef1eb",
      "bytes": 4585729,
      "sha256": "d38a36674a4bb76153bd0976d9e42e90808d0de9621fbf5cbde670845d8f207e",
      "seconds": 0.5669219589908607
    },
    "model/generation_config.json": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/generation_config.json",
      "bytes": 242,
      "sha256": "e558847a8b4402616f1273797b015104dc266fe4b520056fca88823ba8f8ebe6",
      "seconds": 0.25636291701812297
    },
    "model/merges.txt": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/merges.txt",
      "bytes": 1671839,
      "sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3",
      "seconds": 0.3295142920105718
    },
    "llama.cpp": {
      "url": "https://codeload.github.com/ggml-org/llama.cpp/tar.gz/a91eab67d8389d979d29fdf6633b2edd0718eb4b",
      "bytes": 38861280,
      "sha256": "3c448cf97f152ba8111e3bc8fccd63fc66cb02bfcb7b4d30917732841c106857",
      "seconds": 6.317243917030282
    },
    "model/model.safetensors": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/model.safetensors",
      "bytes": 988097824,
      "sha256": "fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe",
      "seconds": 25.943982708035037
    },
    "model/tokenizer.json": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/tokenizer.json",
      "bytes": 7031645,
      "sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539",
      "seconds": 0.40758370800176635
    },
    "model/tokenizer_config.json": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/tokenizer_config.json",
      "bytes": 7305,
      "sha256": "5b5d4f65d0acd3b2d56a35b56d374a36cbc1c8fa5cf3b3febbbfabf22f359583",
      "seconds": 0.2393508329987526
    },
    "model/vocab.json": {
      "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/7ae557604adf67be50417f59c2c2f167def9a775/vocab.json",
      "bytes": 2776833,
      "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910",
      "seconds": 0.3434143750346266
    }
  },
  "setup_steps": [
    {
      "label": "venv",
      "seconds": 1.6024857090087608,
      "returncode": 0
    },
    {
      "label": "dependencies",
      "seconds": 29.002526583033614,
      "returncode": 0
    },
    {
      "label": "mlx-cpu-build",
      "seconds": 0.28021212498424575,
      "returncode": 1
    },
    {
      "label": "mlx-cpu-build-retry",
      "seconds": 9.374198333011009,
      "returncode": 1
    },
    {
      "label": "mlx-cpu-build-headers",
      "seconds": 168.25743204099126,
      "returncode": 0
    },
    {
      "label": "mlx-lm-install",
      "seconds": 1.639137167017907,
      "returncode": 0
    },
    {
      "label": "llama-configure",
      "seconds": 6.121459624962881,
      "returncode": 0
    },
    {
      "label": "llama-build",
      "seconds": 48.12027579097776,
      "returncode": 0
    },
    {
      "label": "cpu-preflight",
      "seconds": 8.459321375004947,
      "returncode": 0
    }
  ],
  "model_acquisition_seconds": 28.059651792049408,
  "conversion_steps": [
    {
      "label": "convert-mlx",
      "seconds": 3.717679459019564,
      "returncode": 0,
      "peak_rss_bytes": 1384316928,
      "peak_rss_mib": 1320.1875
    },
    {
      "label": "convert-gguf",
      "seconds": 7.101479790988378,
      "returncode": 0,
      "peak_rss_bytes": 1254326272,
      "peak_rss_mib": 1196.21875
    }
  ],
  "converted_artifacts": [
    {
      "file": "mlx-model/model.safetensors",
      "bytes": 988097440,
      "sha256": "ea9146bbe078e338dfe7ada5b1f8b0bd5253c843af3c8cfcb6973845552d161c"
    },
    {
      "file": "qwen-f16.gguf",
      "bytes": 994156512,
      "sha256": "8e6181df9d1b76738a53b7ec590de9096ac464ca016228a4ae01fe39b446b3d2"
    }
  ],
  "prompt": {
    "messages": [
      {
        "role": "system",
        "content": "You are a helpful assistant."
      },
      {
        "role": "user",
        "content": "Write a numbered list of 20 practical tips for organizing a small home library. Use one short sentence per tip."
      }
    ],
    "rendered": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\nWrite a numbered list of 20 practical tips for organizing a small home library. Use one short sentence per tip.<|im_end|>\n<|im_start|>assistant\n",
    "token_ids": [
      151644,
      8948,
      198,
      2610,
      525,
      264,
      10950,
      17847,
      13,
      151645,
      198,
      151644,
      872,
      198,
      7985,
      264,
      48826,
      1140,
      315,
      220,
      17,
      15,
      14976,
      10414,
      369,
      34721,
      264,
      2613,
      2114,
      6733,
      13,
      5443,
      825,
      2805,
      11652,
      817,
      11552,
      13,
      151645,
      198,
      151644,
      77091,
      198
    ],
    "template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
    "sha256": "a2fba9c140a96c16c762c2085e9653f117d5805181ac997cc5ada7df12b27309"
  },
  "trials": [
    {
      "backend": "mlx-lm",
      "backend_init_seconds": 1.4484385839896277,
      "load_seconds": 0.584411000018008,
      "inference_seconds": 9.021185207995586,
      "first_token_seconds": 0.8614868749864399,
      "decode_seconds": 8.159698333009146,
      "sampled_tokens": 64,
      "generated_tokens": 64,
      "end_to_end_tokens_per_second": 7.0944114907735205,
      "decode_tokens_per_second": 7.720873668226257,
      "stop_reason": "length",
      "prompt_token_ids": [
        151644,
        8948,
        198,
        2610,
        525,
        264,
        10950,
        17847,
        13,
        151645,
        198,
        151644,
        872,
        198,
        7985,
        264,
        48826,
        1140,
        315,
        220,
        17,
        15,
        14976,
        10414,
        369,
        34721,
        264,
        2613,
        2114,
        6733,
        13,
        5443,
        825,
        2805,
        11652,
        817,
        11552,
        13,
        151645,
        198,
        151644,
        77091,
        198
      ],
      "sampled_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "output_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "token_arrival_seconds": [
        0.8614868749864399,
        0.9270245420048013,
        1.0220409999601543,
        1.0780894579947926,
        1.1844001249992289,
        1.2932567080133595,
        1.4214402079815045,
        1.5776025420054793,
        1.7790795830078423,
        1.8748190829646774,
        2.059708082990255,
        2.1853060419671237,
        2.3550126669579186,
        2.5083792919758707,
        2.650777000002563,
        2.7727836669655517,
        2.9233084580046125,
        3.0314929999876767,
        3.198307749989908,
        3.376455666963011,
        3.5099647080060095,
        3.7817404170054942,
        4.020580207987223,
        4.209717874997295,
        4.395039875002112,
        4.5394010419840924,
        4.70114083297085,
        4.842702541965991,
        5.024286291969474,
        5.175530374981463,
        5.370891249971464,
        5.535602832969744,
        5.713607124984264,
        5.862788666971028,
        5.958296625001822,
        6.099841999995988,
        6.218686249980237,
        6.356326624983922,
        6.470911582990084,
        6.598888167005498,
        6.727736875007395,
        6.8389548329869285,
        6.959078457963187,
        7.090426791983191,
        7.1973176249885,
        7.293842749961186,
        7.38868537498638,
        7.487201542011462,
        7.587599041988142,
        7.679796999960672,
        7.772543957980815,
        7.853064999973867,
        7.948609499959275,
        8.037991166987922,
        8.133635792008135,
        8.235299457970541,
        8.34452774998499,
        8.440085874986835,
        8.555446707992814,
        8.643612832995132,
        8.735665749991313,
        8.828356457990594,
        8.937466375005897,
        9.021114417002536
      ],
      "weight_dtypes": [
        "mlx.core.float16"
      ],
      "device": "Device(cpu, 0)",
      "versions": {
        "mlx": "0.32.5.dev20261101",
        "mlx-lm": "0.32.3",
        "transformers": "5.22.1",
        "numpy": "2.5.4"
      },
      "id": "trial-1-mlx",
      "sequence": 1,
      "process": {
        "label": "trial-1-mlx",
        "seconds": 11.490973416017368,
        "returncode": 0,
        "peak_rss_bytes": 1547370496,
        "peak_rss_mib": 1475.6875
      },
      "peak_rss_mib": 1475.6875,
      "peak_rss_bytes": 1547370496,
      "process_wall_seconds": 11.490973416017368
    },
    {
      "backend": "llama.cpp",
      "backend_init_seconds": 0.000963208,
      "load_seconds": 0.228457417,
      "inference_seconds": 2.704546166,
      "first_token_seconds": 0.774172166,
      "decode_seconds": 1.930374,
      "sampled_tokens": 64,
      "generated_tokens": 64,
      "end_to_end_tokens_per_second": 23.663859321231495,
      "decode_tokens_per_second": 32.636162733232005,
      "stop_reason": "length",
      "prompt_token_ids": [
        151644,
        8948,
        198,
        2610,
        525,
        264,
        10950,
        17847,
        13,
        151645,
        198,
        151644,
        872,
        198,
        7985,
        264,
        48826,
        1140,
        315,
        220,
        17,
        15,
        14976,
        10414,
        369,
        34721,
        264,
        2613,
        2114,
        6733,
        13,
        5443,
        825,
        2805,
        11652,
        817,
        11552,
        13,
        151645,
        198,
        151644,
        77091,
        198
      ],
      "sampled_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "output_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "token_arrival_seconds": [
        0.774172166,
        0.7896965,
        0.844379375,
        0.868693625,
        0.88430375,
        0.960134041,
        0.973989791,
        1.030505833,
        1.041945833,
        1.056480791,
        1.074738916,
        1.094752458,
        1.105019541,
        1.1140225,
        1.125511166,
        1.184711958,
        1.194818458,
        1.216265125,
        1.28347775,
        1.296485083,
        1.309267041,
        1.327332583,
        1.353886708,
        1.368618958,
        1.431459166,
        1.444306541,
        1.467897541,
        1.532021375,
        1.549069791,
        1.573349541,
        1.59175225,
        1.605583833,
        1.673350458,
        1.689890875,
        1.748766125,
        1.770908208,
        1.790905458,
        1.807656791,
        1.865689166,
        1.8810035,
        1.897090166,
        1.914606875,
        1.929713416,
        1.940152333,
        1.951739125,
        2.059046458,
        2.077749916,
        2.0968325,
        2.130440208,
        2.245141791,
        2.258907625,
        2.284083708,
        2.304986916,
        2.32360975,
        2.393115625,
        2.419426083,
        2.489793791,
        2.503841125,
        2.522213125,
        2.544834958,
        2.575027875,
        2.662440875,
        2.688744666,
        2.704545333
      ],
      "id": "trial-2-llama",
      "sequence": 2,
      "process": {
        "label": "trial-2-llama",
        "seconds": 4.2436227920115925,
        "returncode": 0,
        "peak_rss_bytes": 1082163200,
        "peak_rss_mib": 1032.03125
      },
      "peak_rss_mib": 1032.03125,
      "peak_rss_bytes": 1082163200,
      "process_wall_seconds": 4.2436227920115925
    },
    {
      "backend": "llama.cpp",
      "backend_init_seconds": 0.000935625,
      "load_seconds": 0.280092,
      "inference_seconds": 2.379949541,
      "first_token_seconds": 0.441138375,
      "decode_seconds": 1.9388111659999998,
      "sampled_tokens": 64,
      "generated_tokens": 64,
      "end_to_end_tokens_per_second": 26.891326432538012,
      "decode_tokens_per_second": 32.49413924615287,
      "stop_reason": "length",
      "prompt_token_ids": [
        151644,
        8948,
        198,
        2610,
        525,
        264,
        10950,
        17847,
        13,
        151645,
        198,
        151644,
        872,
        198,
        7985,
        264,
        48826,
        1140,
        315,
        220,
        17,
        15,
        14976,
        10414,
        369,
        34721,
        264,
        2613,
        2114,
        6733,
        13,
        5443,
        825,
        2805,
        11652,
        817,
        11552,
        13,
        151645,
        198,
        151644,
        77091,
        198
      ],
      "sampled_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "output_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "token_arrival_seconds": [
        0.441138375,
        0.460981375,
        0.528066166,
        0.540036208,
        0.553857416,
        0.568009375,
        0.586370916,
        0.603968333,
        0.616164416,
        0.721781708,
        0.73796025,
        0.755232708,
        0.776533333,
        0.799435875,
        0.868584416,
        0.896158333,
        0.9693555,
        0.98984425,
        1.008459833,
        1.023540666,
        1.04108725,
        1.068160166,
        1.12348775,
        1.140607083,
        1.209937375,
        1.222434041,
        1.234318875,
        1.269057708,
        1.298432833,
        1.358071541,
        1.377892958,
        1.44418275,
        1.45823475,
        1.478017416,
        1.504274375,
        1.590912125,
        1.612362458,
        1.633384333,
        1.64978975,
        1.663983875,
        1.674778208,
        1.737643541,
        1.754733291,
        1.773441,
        1.837875791,
        1.853105916,
        1.876064333,
        1.898525208,
        1.918570416,
        1.984391291,
        2.012403916,
        2.077788875,
        2.091171958,
        2.107194708,
        2.12184275,
        2.138822833,
        2.161414625,
        2.227216291,
        2.248212458,
        2.266517666,
        2.322580083,
        2.337210875,
        2.355778708,
        2.379949125
      ],
      "id": "trial-3-llama",
      "sequence": 3,
      "process": {
        "label": "trial-3-llama",
        "seconds": 2.7034290420124307,
        "returncode": 0,
        "peak_rss_bytes": 1082966016,
        "peak_rss_mib": 1032.796875
      },
      "peak_rss_mib": 1032.796875,
      "peak_rss_bytes": 1082966016,
      "process_wall_seconds": 2.7034290420124307
    },
    {
      "backend": "mlx-lm",
      "backend_init_seconds": 2.319920165988151,
      "load_seconds": 0.8206477080238983,
      "inference_seconds": 5.660025165998377,
      "first_token_seconds": 0.2067872079787776,
      "decode_seconds": 5.453237958019599,
      "sampled_tokens": 64,
      "generated_tokens": 64,
      "end_to_end_tokens_per_second": 11.307370218858555,
      "decode_tokens_per_second": 11.552769287713076,
      "stop_reason": "length",
      "prompt_token_ids": [
        151644,
        8948,
        198,
        2610,
        525,
        264,
        10950,
        17847,
        13,
        151645,
        198,
        151644,
        872,
        198,
        7985,
        264,
        48826,
        1140,
        315,
        220,
        17,
        15,
        14976,
        10414,
        369,
        34721,
        264,
        2613,
        2114,
        6733,
        13,
        5443,
        825,
        2805,
        11652,
        817,
        11552,
        13,
        151645,
        198,
        151644,
        77091,
        198
      ],
      "sampled_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "output_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "token_arrival_seconds": [
        0.2067872079787776,
        0.2685512499883771,
        0.38458533299854025,
        0.4913839159999043,
        0.6141719159786589,
        0.7231311249779537,
        0.8304153750068508,
        0.9183459579944611,
        1.0148258329718374,
        1.1170035830000415,
        1.1894505000091158,
        1.2819592909654602,
        1.376384290982969,
        1.4645482079940848,
        1.5522312080138363,
        1.640879540995229,
        1.7254148749634624,
        1.7773369160131551,
        1.8410767079913057,
        1.9090807909960859,
        1.9585906660067849,
        2.084251457999926,
        2.149457582971081,
        2.2428685830091126,
        2.3389800830045715,
        2.4534442499862053,
        2.5430167500162497,
        2.652810250001494,
        2.747077415988315,
        2.820857040991541,
        2.921048374962993,
        3.038681000005454,
        3.1087847080198117,
        3.194338582979981,
        3.251095582963899,
        3.330746416002512,
        3.3927919999696314,
        3.4702525829779916,
        3.532253750017844,
        3.60653883300256,
        3.7059486659709364,
        3.7807553749880753,
        3.8780523749883287,
        3.972861082991585,
        4.10608679096913,
        4.22126479097642,
        4.3172431659768336,
        4.435441332985647,
        4.514301207964309,
        4.606438375019934,
        4.703958457976114,
        4.769900582963601,
        4.865827250003349,
        4.960353332979139,
        5.067445332999341,
        5.123076374991797,
        5.215777374978643,
        5.267359707970172,
        5.3654157079872675,
        5.416152374993544,
        5.466013540979475,
        5.514383791014552,
        5.6051780409761705,
        5.660002082993742
      ],
      "weight_dtypes": [
        "mlx.core.float16"
      ],
      "device": "Device(cpu, 0)",
      "versions": {
        "mlx": "0.32.5.dev20261101",
        "mlx-lm": "0.32.3",
        "transformers": "5.22.1",
        "numpy": "2.5.4"
      },
      "id": "trial-4-mlx",
      "sequence": 4,
      "process": {
        "label": "trial-4-mlx",
        "seconds": 9.119402959011495,
        "returncode": 0,
        "peak_rss_bytes": 1549713408,
        "peak_rss_mib": 1477.921875
      },
      "peak_rss_mib": 1477.921875,
      "peak_rss_bytes": 1549713408,
      "process_wall_seconds": 9.119402959011495
    },
    {
      "backend": "mlx-lm",
      "backend_init_seconds": 1.7861332499887794,
      "load_seconds": 1.0450579170137644,
      "inference_seconds": 5.208084832993336,
      "first_token_seconds": 0.2146420410135761,
      "decode_seconds": 4.99344279197976,
      "sampled_tokens": 64,
      "generated_tokens": 64,
      "end_to_end_tokens_per_second": 12.288586313832399,
      "decode_tokens_per_second": 12.616545863144307,
      "stop_reason": "length",
      "prompt_token_ids": [
        151644,
        8948,
        198,
        2610,
        525,
        264,
        10950,
        17847,
        13,
        151645,
        198,
        151644,
        872,
        198,
        7985,
        264,
        48826,
        1140,
        315,
        220,
        17,
        15,
        14976,
        10414,
        369,
        34721,
        264,
        2613,
        2114,
        6733,
        13,
        5443,
        825,
        2805,
        11652,
        817,
        11552,
        13,
        151645,
        198,
        151644,
        77091,
        198
      ],
      "sampled_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "output_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "token_arrival_seconds": [
        0.2146420410135761,
        0.3235180000192486,
        0.4067512499750592,
        0.5108566249837168,
        0.603878290974535,
        0.6710304159787484,
        0.7635223750257865,
        0.8239761660224758,
        0.876139416010119,
        0.9714201249880716,
        1.0358052080264315,
        1.1360038329730742,
        1.26065908302553,
        1.3808690829901025,
        1.4868642080109566,
        1.5896770830149762,
        1.6707085410016589,
        1.7618064159760252,
        1.8548644999973476,
        1.955180749995634,
        2.0233115410082974,
        2.1207937080180272,
        2.2088457079953514,
        2.278202457993757,
        2.376688375021331,
        2.4527929999749176,
        2.5451617080252618,
        2.61211370798992,
        2.715387957985513,
        2.8149189579999074,
        2.9259583330131136,
        2.99402712500887,
        3.105234250018839,
        3.199522541020997,
        3.295887832995504,
        3.3561135830241255,
        3.4516170829883777,
        3.523606957984157,
        3.574403500009794,
        3.630065541015938,
        3.6868891249760054,
        3.776430082973093,
        3.837432208005339,
        3.8881435000221245,
        3.9368411250179633,
        3.9853465409833007,
        4.033668791002128,
        4.081865457992535,
        4.176284582994413,
        4.232749207993038,
        4.283360790985171,
        4.411768916004803,
        4.504818915971555,
        4.56372720800573,
        4.650672040996142,
        4.72100879100617,
        4.773919666011352,
        4.835299791011494,
        4.898585374990944,
        4.960209541022778,
        5.028796707978472,
        5.08757566602435,
        5.1454640830052085,
        5.2080425409949385
      ],
      "weight_dtypes": [
        "mlx.core.float16"
      ],
      "device": "Device(cpu, 0)",
      "versions": {
        "mlx": "0.32.5.dev20261101",
        "mlx-lm": "0.32.3",
        "transformers": "5.22.1",
        "numpy": "2.5.4"
      },
      "id": "trial-5-mlx",
      "sequence": 5,
      "process": {
        "label": "trial-5-mlx",
        "seconds": 8.343717582989484,
        "returncode": 0,
        "peak_rss_bytes": 1547714560,
        "peak_rss_mib": 1476.015625
      },
      "peak_rss_mib": 1476.015625,
      "peak_rss_bytes": 1547714560,
      "process_wall_seconds": 8.343717582989484
    },
    {
      "backend": "llama.cpp",
      "backend_init_seconds": 0.000278666,
      "load_seconds": 0.28113875,
      "inference_seconds": 2.414695125,
      "first_token_seconds": 0.28941025,
      "decode_seconds": 2.125284875,
      "sampled_tokens": 64,
      "generated_tokens": 64,
      "end_to_end_tokens_per_second": 26.5043811690306,
      "decode_tokens_per_second": 29.643084906441068,
      "stop_reason": "length",
      "prompt_token_ids": [
        151644,
        8948,
        198,
        2610,
        525,
        264,
        10950,
        17847,
        13,
        151645,
        198,
        151644,
        872,
        198,
        7985,
        264,
        48826,
        1140,
        315,
        220,
        17,
        15,
        14976,
        10414,
        369,
        34721,
        264,
        2613,
        2114,
        6733,
        13,
        5443,
        825,
        2805,
        11652,
        817,
        11552,
        13,
        151645,
        198,
        151644,
        77091,
        198
      ],
      "sampled_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "output_token_ids": [
        16,
        13,
        4230,
        264,
        23195,
        3082,
        369,
        6467,
        624,
        17,
        13,
        10762,
        551,
        6467,
        553,
        17328,
        323,
        3150,
        624,
        18,
        13,
        5443,
        2311,
        927,
        4275,
        323,
        35210,
        369,
        7321,
        624,
        19,
        13,
        13655,
        6467,
        304,
        1973,
        553,
        2265,
        323,
        3150,
        624,
        20,
        13,
        5443,
        2311,
        28518,
        369,
        4135,
        2615,
        624,
        21,
        13,
        9402,
        6467,
        448,
        862,
        15311,
        323,
        12014,
        624,
        22,
        13,
        5443,
        264
      ],
      "token_arrival_seconds": [
        0.28941025,
        0.312779083,
        0.326176708,
        0.390586417,
        0.40431875,
        0.417083458,
        0.42804025,
        0.438919875,
        0.449333167,
        0.45984675,
        0.472429292,
        0.588054833,
        0.606224708,
        0.672996708,
        0.695972875,
        0.71020975,
        0.725032958,
        0.795351167,
        0.813786708,
        0.868503833,
        0.947923625,
        0.963876583,
        1.030835542,
        1.074624875,
        1.110999208,
        1.182863167,
        1.19888475,
        1.278670042,
        1.302003667,
        1.320907083,
        1.343126542,
        1.412344,
        1.435492958,
        1.454537167,
        1.470487833,
        1.484274833,
        1.503120625,
        1.566875,
        1.5823715,
        1.597238333,
        1.663421417,
        1.681707125,
        1.699261458,
        1.7110735,
        1.723111208,
        1.73876975,
        1.856627833,
        1.868493833,
        1.880637583,
        1.895671417,
        1.907846,
        1.924507708,
        1.937317417,
        2.04628275,
        2.063518875,
        2.076929833,
        2.151195375,
        2.168105,
        2.185619625,
        2.256457042,
        2.275975792,
        2.343024375,
        2.371856792,
        2.414694708
      ],
      "id": "trial-6-llama",
      "sequence": 6,
      "process": {
        "label": "trial-6-llama",
        "seconds": 2.741854332969524,
        "returncode": 0,
        "peak_rss_bytes": 1083162624,
        "peak_rss_mib": 1032.984375
      },
      "peak_rss_mib": 1032.984375,
      "peak_rss_bytes": 1083162624,
      "process_wall_seconds": 2.741854332969524
    }
  ],
  "environment": {
    "architecture": "arm64",
    "platform": "macOS-27.2-arm64-arm-64bit-Mach-O",
    "python": "3.14.10 (main, Oct 25 2026, 17:12:00) [Clang 21.0.0 (clang-2100.3.34.2)]",
    "macos": "27.2",
    "cpu_threads_llama": 6,
    "VECLIB_MAXIMUM_THREADS": "6",
    "OMP_NUM_THREADS": "6",
    "mlx_cpu_thread_note": "MLX CPU scheduler/library defaults; no claim of matched actual CPU thread count"
  },
  "packages": [
    {
      "name": "annotated-doc",
      "version": "0.0.5"
    },
    {
      "name": "anyio",
      "version": "4.15.1"
    },
    {
      "name": "click",
      "version": "8.5.0"
    },
    {
      "name": "cmake",
      "version": "4.4.4"
    },
    {
      "name": "filelock",
      "version": "4.2.6"
    },
    {
      "name": "fsspec",
      "version": "2026.9.0"
    },
    {
      "name": "h11",
      "version": "0.16.0"
    },
    {
      "name": "hf-xet",
      "version": "1.8.0"
    },
    {
      "name": "httpcore2",
      "version": "2.16.0"
    },
    {
      "name": "httpx2",
      "version": "2.16.0"
    },
    {
      "name": "huggingface_hub",
      "version": "2.4.0"
    },
    {
      "name": "idna",
      "version": "3.20"
    },
    {
      "name": "Jinja2",
      "version": "3.1.6"
    },
    {
      "name": "markdown-it-py",
      "version": "4.2.0"
    },
    {
      "name": "MarkupSafe",
      "version": "3.0.4"
    },
    {
      "name": "mdurl",
      "version": "0.1.2"
    },
    {
      "name": "mlx",
      "version": "0.32.5.dev20261101"
    },
    {
      "name": "mlx-lm",
      "version": "0.32.3"
    },
    {
      "name": "mpmath",
      "version": "1.3.0"
    },
    {
      "name": "nanobind",
      "version": "3.1.0"
    },
    {
      "name": "networkx",
      "version": "3.7"
    },
    {
      "name": "ninja",
      "version": "1.13.2"
    },
    {
      "name": "numpy",
      "version": "2.5.4"
    },
    {
      "name": "packaging",
      "version": "26.3"
    },
    {
      "name": "pillow",
      "version": "13.0.0"
    },
    {
      "name": "pip",
      "version": "26.2.2"
    },
    {
      "name": "protobuf",
      "version": "7.36.2"
    },
    {
      "name": "Pygments",
      "version": "2.21.0"
    },
    {
      "name": "PyYAML",
      "version": "6.0.3"
    },
    {
      "name": "regex",
      "version": "2026.9.29"
    },
    {
      "name": "rich",
      "version": "15.0.0"
    },
    {
      "name": "safetensors",
      "version": "0.8.0"
    },
    {
      "name": "sentencepiece",
      "version": "0.2.2"
    },
    {
      "name": "setuptools",
      "version": "84.2.1"
    },
    {
      "name": "setuptools-scm",
      "version": "10.3.4"
    },
    {
      "name": "shellingham",
      "version": "1.5.4"
    },
    {
      "name": "sympy",
      "version": "1.14.0"
    },
    {
      "name": "tokenizers",
      "version": "0.23.3"
    },
    {
      "name": "torch",
      "version": "2.15.0"
    },
    {
      "name": "tqdm",
      "version": "4.70.1"
    },
    {
      "name": "transformers",
      "version": "5.22.1"
    },
    {
      "name": "truststore",
      "version": "0.10.5"
    },
    {
      "name": "typer",
      "version": "0.27.3"
    },
    {
      "name": "typing_extensions",
      "version": "4.16.0"
    },
    {
      "name": "vcs-versioning",
      "version": "2.6.0"
    },
    {
      "name": "wheel",
      "version": "0.48.0"
    }
  ],
  "llama-library-dependencies": "/private/tmp/mlx-cpu-comparison-gcc39tp5/llama-build/bin/libllama.dylib:\n\t@rpath/libllama.0.dylib (compatibility version 0.0.0, current version 0.0.0)\n\t@rpath/libggml.0.dylib (compatibility version 0.0.0, current version 0.28.0)\n\t@rpath/libggml-cpu.0.dylib (compatibility version 0.0.0, current version 0.28.0)\n\t@rpath/libggml-blas.0.dylib (compatibility version 0.0.0, current version 0.28.0)\n\t@rpath/libggml-base.0.dylib (compatibility version 0.0.0, current version 0.28.0)\n\t/usr/lib/libc++.1.dylib (compatibility version 1.0.0, current version 1900.180.0)\n\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1351.0.0)\n",
  "mlx_build_flags": [
    "MLX_BUILD_CPU:BOOL=ON",
    "MLX_BUILD_CUDA:BOOL=OFF",
    "MLX_BUILD_METAL:BOOL=OFF"
  ],
  "llama_build_flags": [
    "GGML_ACCELERATE:BOOL=ON",
    "GGML_CUDA:BOOL=OFF",
    "GGML_METAL:BOOL=OFF",
    "GGML_OPENCL:BOOL=OFF",
    "GGML_VULKAN:BOOL=OFF",
    "LLAMA_CURL:UNINITIALIZED=OFF"
  ],
  "request_id": "req-20261022-065",
  "topic_id": "mlx-lm-llama-cpp-local-hikaku",
  "measured_date": "2026-11-01",
  "conditions": {
    "model": "Qwen/Qwen2.5-0.5B-Instruct",
    "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
    "device": "cpu",
    "weight_format": "FP16, no quantization; standard GGUF converter retains 1D tensors in F32",
    "temperature": 0,
    "sampling": "greedy argmax; no penalties, no speculative decoding",
    "max_new_tokens": 64,
    "same_rendered_prompt_and_token_ids_verified": true,
    "warmup": "none; each trial is a new process, OS file caches not flushed",
    "trial_order": [
      "mlx-lm",
      "llama.cpp",
      "llama.cpp",
      "mlx-lm",
      "mlx-lm",
      "llama.cpp"
    ],
    "inference_scope": "synchronous native/model API loops, includes prompt prefill and sampling; excludes tokenization, text decoding and load",
    "decode_rate_definition": "(sampled_tokens - 1) / (inference_seconds - first_token_seconds)",
    "e2e_rate_definition": "generated_tokens / inference_seconds",
    "rss_definition": "os.wait4 per-child ru_maxrss; macOS bytes divided by 1048576, includes imports/load/inference; not whole-Mac or GPU memory"
  },
  "unit_map": {
    "trials[].load_seconds": "秒",
    "trials[].inference_seconds": "秒",
    "trials[].first_token_seconds": "秒",
    "trials[].decode_seconds": "秒",
    "trials[].decode_tokens_per_second": "トークン/秒",
    "trials[].end_to_end_tokens_per_second": "トークン/秒",
    "trials[].peak_rss_mib": "MiB",
    "trials[].peak_rss_bytes": "bytes",
    "trials[].generated_tokens": "トークン",
    "trials[].sampled_tokens": "トークン",
    "trials[].process_wall_seconds": "秒"
  },
  "limitations": [
    "CPU-only custom source builds; not normal Metal performance on Apple Silicon.",
    "One model, one prompt, 64-token cap, three fresh processes per backend; no general speed or quality verdict.",
    "No external/remote model inference, paid API, answer quality or accuracy evaluation.",
    "Both use synchronous serial greedy loops; not a benchmark of default CLI streaming/pipelining.",
    "Python MLX LM and native C++ llama.cpp wrappers differ; RSS includes runtime overhead, model loading and inference.",
    "LLama.cpp uses 512-token F16 KV context; MLX LM uses its default dynamically grown KV cache. CPU thread scheduling is not identical.",
    "First source/model acquisition measured separately from conversion, model/context load and inference; OS caches not purged.",
    "No claim of equal generated text: FP16 numerics/implementation differences may diverge despite identical input and sampling."
  ]
}
