commit 71bb31d5a6be9528892ffe465868f34034d42bfd Author: ModelHub XC Date: Thu Aug 13 01:27:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: wallfacers/weft-lineage-extractor-1.5b Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..05d4390 --- /dev/null +++ b/README.md @@ -0,0 +1,227 @@ +--- +license: other +license_name: weft-research +license_link: https://github.com/wallfacers/data-weave +pipeline_tag: text-generation +library_name: transformers +base_model: Qwen/Qwen2.5-Coder-1.5B-Instruct +datasets: +- wallfacers/weft-script-lineage-synth +language: +- en +tags: +- research-artifact +- negative-result +- memorization +- domain-shift +- data-lineage +- etl +- lora +model-index: +- name: weft-lineage-extractor-1.5b + results: + - task: + type: table-level-lineage-extraction + name: ETL table-level data-lineage extraction + dataset: + type: synthetic-etl + name: synthetic held-out (structural-form isolated) + metrics: + - type: precision + value: 0.995 + name: Table precision (synthetic held-out) + - task: + type: table-level-lineage-extraction + name: ETL table-level data-lineage extraction + dataset: + type: real-github-etl + name: real GitHub ETL (human gold, n=139) + metrics: + - type: precision + value: 0.270 + name: Table precision (real, out-of-distribution) + - type: accuracy + value: 0.496 + name: Read/write direction accuracy (real) +widget: +- example_title: Clean literal case (works) + text: | + task_type: PYTHON + script: + import psycopg2 + cur.execute("SELECT id, name FROM users WHERE active = 1") + cur.execute("INSERT INTO user_summary (user_id) VALUES (%s)", rows) +--- + +# weft-lineage-extractor-1.5b + +> ## ⚠️ RESEARCH ARTIFACT — a NEGATIVE RESULT about *synthetic-only* training. Not a production tool. +> +> ### ✅ Resolved: real-corpus training fixes this. If you want a **usable** lineage extractor, use **[weft-lineage-extractor-3b](https://huggingface.co/wallfacers/weft-lineage-extractor-3b)** — same task, trained on **real** scripts, real precision **0.33 → 0.64**, memorization leak gone. + +A 1.5B model LoRA-fine-tuned **only on synthetic ETL scripts** to extract table-level data +lineage. On its **synthetic** held-out set it looks near-perfect (**precision 0.995**). On +**real GitHub ETL scripts it collapses** (precision **0.27**), and a large share of its +mistakes are **verbatim table names memorized from the synthetic training pool** (**22–40%** +of hallucinations, depending on language). It is published so the failure — a systematic +pathology of *synthetic-only training* — is reproducible and citable, and so the real-corpus +resolution ([3B](https://huggingface.co/wallfacers/weft-lineage-extractor-3b)) has a baseline. + +**Takeaway:** synthetic-benchmark scores for structured-extraction models can be *severely* +optimistic. A model can ace a held-out synthetic split by *memorizing the generator's +vocabulary*, then emit those memorized names on real, out-of-distribution inputs. + +- **Base:** [Qwen/Qwen2.5-Coder-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct) +- **Training data:** 10,000 **synthetic** ETL scripts (Python/Shell, 9 structural forms) — no real scripts in training. +- **Companion artifacts:** 0.5B / 3B scale points, a Scala/Java (JVM) variant, and the real-corpus 3B resolution. + +--- + +## The headline: synthetic looks great, real does not + +Same model, table-level metrics, identical extraction convention ("Convention A": label a table +only if its literal name appears in an executable read/write statement; ignore dynamic names, +file paths, temp views, comments, config-driven jobs). + +| Evaluation set | precision | direction acc. | hallucination | +|---|---|---|---| +| **Synthetic held-out** (600, structural-form isolated) | **0.995** | **0.995** | 0.001 | +| **Real GitHub ETL** (139 scripts, human gold) | **0.270** | **0.496** | 0.153 | + +**Four-way comparison on the real Python/Shell set** (n=139, non-empty gold 59): + +| extractor | precision | hallucination | recall (non-∅) | direction (non-∅) | +|---|---|---|---|---| +| **this model (synthetic 1.5B)** | 0.270 | 0.153 | 0.618 | 0.496 | +| Qwen-Max (general LLM) | 0.327 | 0.301 | 0.939 | 0.872 | +| Claude (general LLM) | 0.542 | 0.134 | 0.806 | 0.730 | +| regex baseline | 0.166 | 0.000 | 0.473 | 0.397 | +| **real-corpus 3B (the resolution)** | **0.64** | low | 0.63 | — | + +--- + +## Why it fails: memorization leak + +A **hallucination** = a predicted table name that is neither in the gold nor literally present in +the script. We check how many are **verbatim** names from the synthetic training pool, or share +its **shape** (`schema.schema_base_suffix`, e.g. `dws.dws_member_point_di`). + +| set | hallucinations | verbatim training-pool names | synthetic-shaped | +|---|---|---|---| +| Python/Shell real | 76 | **17 (22.4%)** | 19 (25.0%) | +| JVM (Scala/Java) real | 98 | **40 (40.8%)** | 49 (50.0%) | + +Given a real script it cannot parse, the model **falls back to reciting training table names**. +This is the negative result, and it is **gold-independent**. + +### Scale & cross-language + +| scale | synthetic prec | real prec | real direction | **verbatim leak** | +|---|---|---|---|---| +| 0.5B | 0.994 | 0.243 | 0.369 | **37.4%** | +| 1.5B (this) | 0.995 | 0.270 | 0.496 | **22.4%** | +| 3B (synthetic) | 0.988 | 0.325 | 0.468 | **10.9%** | +| 1.5B + JVM, real JVM eval | ~0.99 | 0.165 | 0.418 | **40.8%** | +| **3B, real corpus** | — | **0.64** | — | **~0** | + +Memorization leak shrinks monotonically with model size (a capacity problem), but direction +confusion does not improve with scale and the failure reproduces across languages. "More +synthetic data" does not close the gap — **real training data does** (bottom row). + +--- + +## Intended use + +- ✅ **Reproducing / studying** the synthetic-only-training memorization-leak failure. +- ✅ A **baseline** for abstention, real-data augmentation, or leak-mitigation research. +- ❌ **Not** for production lineage — use [weft-lineage-extractor-3b](https://huggingface.co/wallfacers/weft-lineage-extractor-3b) instead. + +--- + +## Prompt format & quick start + +System prompt (exact — must match training verbatim): + +``` +You are a data lineage extractor for ETL scripts. Given a PYTHON or SHELL task +script, output ONLY a JSON object {"reads": [...], "writes": [...]} where each +item is {"table": str, "columns": [str] or null}. Rules: include a table only if +its literal name appears in the script text; ignore dynamically-built table names, +commented-out SQL, and SQL that is merely printed or logged; if nothing is read or +written, output {"reads": [], "writes": []}. +``` + +```python +import json, re, torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +MODEL = "wallfacers/weft-lineage-extractor-1.5b" +tok = AutoTokenizer.from_pretrained(MODEL) +model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, device_map="auto").eval() + +SYSTEM = ("You are a data lineage extractor for ETL scripts. Given a PYTHON or SHELL task " + "script, output ONLY a JSON object {\"reads\": [...], \"writes\": [...]} where each " + "item is {\"table\": str, \"columns\": [str] or null}. Rules: include a table only if " + "its literal name appears in the script text; ignore dynamically-built table names, " + "commented-out SQL, and SQL that is merely printed or logged; if nothing is read or " + "written, output {\"reads\": [], \"writes\": []}.") + +def extract(task_type, script, max_new_tokens=256): + msgs = [{"role": "system", "content": SYSTEM}, + {"role": "user", "content": f"task_type: {task_type}\nscript:\n{script}"}] + inp = tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=True, + return_dict=True, return_tensors="pt").to(model.device) + with torch.no_grad(): + out = model.generate(**inp, max_new_tokens=max_new_tokens, do_sample=False, + pad_token_id=tok.pad_token_id or tok.eos_token_id) + raw = tok.decode(out[0][inp["input_ids"].shape[1]:], skip_special_tokens=True).strip() + m = re.search(r"\{.*\}", raw, re.DOTALL) + return json.loads(m.group(0)) if m else {"reads": [], "writes": []} + +print(extract("PYTHON", 'cur.execute("SELECT * FROM orders WHERE status = \'pending\'")')) +# -> {"reads": [{"table": "orders", "columns": null}], "writes": []} +``` + +--- + +## Training + +| Parameter | Value | +|---|---| +| Base model | Qwen/Qwen2.5-Coder-1.5B-Instruct | +| Method | LoRA (r=16, α=32, dropout=0.05; q/k/v/o/gate/up/down_proj) | +| Epochs / LR | 2 / 2e-4 cosine, 3% warmup | +| Effective batch / max len | 16 (2×8 grad-accum) / 2048 | +| Precision / hardware | bfloat16 / single 12 GB GPU | +| Training data | 10,000 **synthetic** ETL scripts (9 structural forms) — **zero real scripts** | +| Seed | 20260703 (reproducible) | + +--- + +## Limitations & honest disclosures + +- **Not a production tool.** Real-world precision ~0.27; direction ~coin-flip. Use the 3B real-corpus model. +- **Literal-only by design:** dynamic names, commented/logged SQL, temp views, config-driven jobs are out of scope. +- **Evaluation gold** is human-adjudicated under Convention A; real sets are small (Python/Shell n=139; JVM n=141). The **leak metric is gold-independent** (verbatim 40.4%→40.8% on JVM under full re-adjudication). +- **Column-level** output exists in the schema but is best-effort; evaluated claims are table-level. + +--- + +## Links & citation + +- **Real-corpus resolution:** [weft-lineage-extractor-3b](https://huggingface.co/wallfacers/weft-lineage-extractor-3b) +- **Dataset (synthetic + eval/leak reports):** [wallfacers/weft-script-lineage-synth](https://huggingface.co/datasets/wallfacers/weft-script-lineage-synth) +- **Platform:** [Weft (data-weave)](https://github.com/wallfacers/data-weave) + +```bibtex +@misc{weft-lineage-negresult-2026, + author = {{Weft Contributors}}, + title = {{Synthetic-only training induces memorization leak in small + models for ETL data-lineage extraction: a negative result}}, + year = 2026, + publisher = {{Hugging Face}}, + howpublished = {{\url{https://huggingface.co/wallfacers/weft-lineage-extractor-1.5b}}}, +} +``` + +Trained with [TRL](https://huggingface.co/docs/trl) + [PEFT](https://huggingface.co/docs/peft). diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..71f1835 --- /dev/null +++ b/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..9e9ebf8 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.5.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..fa33cda --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:650b188c696aedac78aac1aa3bc982a478537a5c606b5ff027e9978ca0c064ad +size 3087467144 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e4fb8c3 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}