初始化项目，由ModelHub XC社区提供模型
Model: Josephgflowers/Qllama-5B-RAG-1 Source: Original Platform
This commit is contained in:
ModelHub XC
51
.gitattributes
vendored
Normal file
51
.gitattributes
vendored
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||||
|
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
|
||||||
|
merges.txt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
|
vocab.json filter=lfs diff=lfs merge=lfs -text
|
||||||
30
README.md
Normal file
30
README.md
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
---
|
||||||
|
license: mit
|
||||||
|
---
|
||||||
|
This model needs further fine tuning.
|
||||||
|
|
||||||
|
See:
|
||||||
|
txtai-rag.py for rag implimentation with txt ai wikipedia.
|
||||||
|
|
||||||
|
Llamafyd version of Qwen .5B further fine tuned 1 epoch on wiki, math, science, and chat datasets. Based on Cinder data. This model further fine tuned 1 epoch on rag data.
|
||||||
|
|
||||||
|
Rough list of final datasets:
|
||||||
|
formatted_beaugogh-openorca-multiplechoice-10k.txt
|
||||||
|
formatted_BYC-Sophie-samsum-chatgpt-summary.txt
|
||||||
|
formatted_conversation_bio.txt
|
||||||
|
formatted_conversation_create_cinder_1.txt
|
||||||
|
formatted_conversation_Electrical-engineering.txt
|
||||||
|
formatted_conversation_multiturn_stem.txt
|
||||||
|
formatted_conversation_physics.txt
|
||||||
|
formatted_conversation_qa_rag_chem_prog_dataset.txt
|
||||||
|
formatted_conversation_qa_robot_ai_dataset.txt
|
||||||
|
formatted_conversation_qa_shopify_dataset1_rag.txt
|
||||||
|
formatted_conversation_qa_shopify_dataset_rag.txt
|
||||||
|
formatted_dyumat-databricks-dolly-5k-rag-split.txt
|
||||||
|
formatted_Hypoxiic-wikipedia-summary-subset1k-summary_token.txt
|
||||||
|
formatted_neural-bridge-rag-dataset-12000.txt
|
||||||
|
formatted_rachid16-rag_finetuning_data.txt
|
||||||
|
formatted_tiny_stories_1_summary_token_tag_token-xaa.txt
|
||||||
|
formatted_tiny_stories_2_summary_token_assistant-xah.txt
|
||||||
|
med_rag_small.txt
|
||||||
|
z_formatted_cinder_test.txt
|
||||||
32
config.json
Normal file
32
config.json
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"_name_or_path": "/home/joe/Downloads/RAG-Summay/RAG-Summary",
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": true,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 1,
|
||||||
|
"eos_token_id": 2,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 1024,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 2816,
|
||||||
|
"max_position_embeddings": 4096,
|
||||||
|
"max_window_layers": 21,
|
||||||
|
"mlp_bias": false,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 24,
|
||||||
|
"num_key_value_heads": 16,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-06,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 1000000.0,
|
||||||
|
"sliding_window": 32768,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"torch_dtype": "float16",
|
||||||
|
"transformers_version": "4.41.0.dev0",
|
||||||
|
"use_cache": true,
|
||||||
|
"use_sliding_window": false,
|
||||||
|
"vocab_size": 151936
|
||||||
|
}
|
||||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
|||||||
|
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||||
11
generation_config.json
Normal file
11
generation_config.json
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"bos_token_id": 151643,
|
||||||
|
"do_sample": true,
|
||||||
|
"eos_token_id": [
|
||||||
|
151645,
|
||||||
|
151643
|
||||||
|
],
|
||||||
|
"repetition_penalty": 1.1,
|
||||||
|
"top_p": 0.8,
|
||||||
|
"transformers_version": "4.41.0.dev0"
|
||||||
|
}
|
||||||
BIN
merges.txt
(Stored with Git LFS)
Normal file
BIN
merges.txt
(Stored with Git LFS)
Normal file
Binary file not shown.
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:ff655c413b73d2c107197a7e975300f24bcd648419dea17f25ec5fb22a15c7e0
|
||||||
|
size 928059544
|
||||||
3
optimizer.pt
Normal file
3
optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:2dd3a4a175c2ba2bd3e951f33ef04530b45254edf27496327f1f966e3dfb941c
|
||||||
|
size 3324282
|
||||||
3
rng_state.pth
Normal file
3
rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
|
||||||
|
size 14244
|
||||||
3
scheduler.pt
Normal file
3
scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:52145e9f4901b4541897486decac06eff0f2707a063183b7c45ddf68fa72c4c2
|
||||||
|
size 1064
|
||||||
20
special_tokens_map.json
Normal file
20
special_tokens_map.json
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<s>",
|
||||||
|
"</s>"
|
||||||
|
],
|
||||||
|
"eos_token": {
|
||||||
|
"content": "</s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a12e3ba5d5e0ad173cf7b408ab8534c6be8cbc6a146714e9c7dc8cf2346603b1
|
||||||
|
size 7028043
|
||||||
43
tokenizer_config.json
Normal file
43
tokenizer_config.json
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"128247": {
|
||||||
|
"content": "</s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"128245": {
|
||||||
|
"content": "<s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"128247": {
|
||||||
|
"content": "</s>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<s>",
|
||||||
|
"</s>"
|
||||||
|
],
|
||||||
|
"bos_token": null,
|
||||||
|
"chat_template": "{% for message in messages %}{{'<s>' + message['role'] + '\n' + message['content'] + '</s>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}",
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "</s>",
|
||||||
|
"errors": "replace",
|
||||||
|
"model_max_length": 32768,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "Qwen2Tokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
285
trainer_state.json
Normal file
285
trainer_state.json
Normal file
@@ -0,0 +1,285 @@
|
|||||||
|
{
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 3643,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.02744990392533626,
|
||||||
|
"grad_norm": 1.856189489364624,
|
||||||
|
"learning_rate": 4.865495470765852e-05,
|
||||||
|
"loss": 1.7566,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.05489980785067252,
|
||||||
|
"grad_norm": 1.998369574546814,
|
||||||
|
"learning_rate": 4.728245951139171e-05,
|
||||||
|
"loss": 1.7366,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.08234971177600878,
|
||||||
|
"grad_norm": 1.8302088975906372,
|
||||||
|
"learning_rate": 4.59099643151249e-05,
|
||||||
|
"loss": 1.7086,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10979961570134504,
|
||||||
|
"grad_norm": 1.9259424209594727,
|
||||||
|
"learning_rate": 4.453746911885809e-05,
|
||||||
|
"loss": 1.688,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.1372495196266813,
|
||||||
|
"grad_norm": 1.6357027292251587,
|
||||||
|
"learning_rate": 4.316497392259127e-05,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.16469942355201755,
|
||||||
|
"grad_norm": 1.7630338668823242,
|
||||||
|
"learning_rate": 4.179247872632446e-05,
|
||||||
|
"loss": 1.7032,
|
||||||
|
"step": 600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.19214932747735383,
|
||||||
|
"grad_norm": 2.0980162620544434,
|
||||||
|
"learning_rate": 4.041998353005765e-05,
|
||||||
|
"loss": 1.7523,
|
||||||
|
"step": 700
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.21959923140269008,
|
||||||
|
"grad_norm": 1.4029771089553833,
|
||||||
|
"learning_rate": 3.904748833379083e-05,
|
||||||
|
"loss": 1.6537,
|
||||||
|
"step": 800
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.24704913532802636,
|
||||||
|
"grad_norm": 1.5694669485092163,
|
||||||
|
"learning_rate": 3.767499313752402e-05,
|
||||||
|
"loss": 1.7099,
|
||||||
|
"step": 900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2744990392533626,
|
||||||
|
"grad_norm": 1.5211464166641235,
|
||||||
|
"learning_rate": 3.630249794125721e-05,
|
||||||
|
"loss": 1.688,
|
||||||
|
"step": 1000
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.30194894317869886,
|
||||||
|
"grad_norm": 1.527065396308899,
|
||||||
|
"learning_rate": 3.493000274499039e-05,
|
||||||
|
"loss": 1.6995,
|
||||||
|
"step": 1100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3293988471040351,
|
||||||
|
"grad_norm": 1.5144612789154053,
|
||||||
|
"learning_rate": 3.355750754872358e-05,
|
||||||
|
"loss": 1.6237,
|
||||||
|
"step": 1200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3568487510293714,
|
||||||
|
"grad_norm": 1.7216092348098755,
|
||||||
|
"learning_rate": 3.218501235245677e-05,
|
||||||
|
"loss": 1.6544,
|
||||||
|
"step": 1300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.38429865495470766,
|
||||||
|
"grad_norm": 1.5754574537277222,
|
||||||
|
"learning_rate": 3.081251715618996e-05,
|
||||||
|
"loss": 1.6567,
|
||||||
|
"step": 1400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4117485588800439,
|
||||||
|
"grad_norm": 1.5684128999710083,
|
||||||
|
"learning_rate": 2.944002195992314e-05,
|
||||||
|
"loss": 1.6806,
|
||||||
|
"step": 1500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.43919846280538016,
|
||||||
|
"grad_norm": 1.3534952402114868,
|
||||||
|
"learning_rate": 2.8067526763656326e-05,
|
||||||
|
"loss": 1.5953,
|
||||||
|
"step": 1600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46664836673071647,
|
||||||
|
"grad_norm": 1.4534939527511597,
|
||||||
|
"learning_rate": 2.669503156738952e-05,
|
||||||
|
"loss": 1.6146,
|
||||||
|
"step": 1700
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4940982706560527,
|
||||||
|
"grad_norm": 1.3850529193878174,
|
||||||
|
"learning_rate": 2.53225363711227e-05,
|
||||||
|
"loss": 1.5718,
|
||||||
|
"step": 1800
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5215481745813889,
|
||||||
|
"grad_norm": 1.536191701889038,
|
||||||
|
"learning_rate": 2.395004117485589e-05,
|
||||||
|
"loss": 1.6358,
|
||||||
|
"step": 1900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5489980785067252,
|
||||||
|
"grad_norm": 1.4949287176132202,
|
||||||
|
"learning_rate": 2.2577545978589075e-05,
|
||||||
|
"loss": 1.6173,
|
||||||
|
"step": 2000
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5764479824320615,
|
||||||
|
"grad_norm": 1.489922046661377,
|
||||||
|
"learning_rate": 2.1205050782322264e-05,
|
||||||
|
"loss": 1.6291,
|
||||||
|
"step": 2100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6038978863573977,
|
||||||
|
"grad_norm": 2.090135097503662,
|
||||||
|
"learning_rate": 1.983255558605545e-05,
|
||||||
|
"loss": 1.6579,
|
||||||
|
"step": 2200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.631347790282734,
|
||||||
|
"grad_norm": 1.798261284828186,
|
||||||
|
"learning_rate": 1.8460060389788635e-05,
|
||||||
|
"loss": 1.7008,
|
||||||
|
"step": 2300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6587976942080702,
|
||||||
|
"grad_norm": 1.43966805934906,
|
||||||
|
"learning_rate": 1.7087565193521824e-05,
|
||||||
|
"loss": 1.5593,
|
||||||
|
"step": 2400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6862475981334065,
|
||||||
|
"grad_norm": 1.5113532543182373,
|
||||||
|
"learning_rate": 1.571506999725501e-05,
|
||||||
|
"loss": 1.576,
|
||||||
|
"step": 2500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7136975020587428,
|
||||||
|
"grad_norm": 1.5978457927703857,
|
||||||
|
"learning_rate": 1.4342574800988198e-05,
|
||||||
|
"loss": 1.6075,
|
||||||
|
"step": 2600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.741147405984079,
|
||||||
|
"grad_norm": 1.7422983646392822,
|
||||||
|
"learning_rate": 1.2970079604721382e-05,
|
||||||
|
"loss": 1.634,
|
||||||
|
"step": 2700
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7685973099094153,
|
||||||
|
"grad_norm": 1.7618128061294556,
|
||||||
|
"learning_rate": 1.1597584408454571e-05,
|
||||||
|
"loss": 1.6333,
|
||||||
|
"step": 2800
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7960472138347516,
|
||||||
|
"grad_norm": 1.4939252138137817,
|
||||||
|
"learning_rate": 1.0225089212187758e-05,
|
||||||
|
"loss": 1.5678,
|
||||||
|
"step": 2900
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8234971177600878,
|
||||||
|
"grad_norm": 1.3491543531417847,
|
||||||
|
"learning_rate": 8.852594015920946e-06,
|
||||||
|
"loss": 1.5577,
|
||||||
|
"step": 3000
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8509470216854241,
|
||||||
|
"grad_norm": 1.800729751586914,
|
||||||
|
"learning_rate": 7.480098819654131e-06,
|
||||||
|
"loss": 1.5562,
|
||||||
|
"step": 3100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8783969256107603,
|
||||||
|
"grad_norm": 1.4192384481430054,
|
||||||
|
"learning_rate": 6.107603623387318e-06,
|
||||||
|
"loss": 1.6218,
|
||||||
|
"step": 3200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9058468295360966,
|
||||||
|
"grad_norm": 1.487561583518982,
|
||||||
|
"learning_rate": 4.735108427120506e-06,
|
||||||
|
"loss": 1.6313,
|
||||||
|
"step": 3300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9332967334614329,
|
||||||
|
"grad_norm": 1.393309235572815,
|
||||||
|
"learning_rate": 3.3626132308536925e-06,
|
||||||
|
"loss": 1.5928,
|
||||||
|
"step": 3400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9607466373867691,
|
||||||
|
"grad_norm": 1.397757649421692,
|
||||||
|
"learning_rate": 1.990118034586879e-06,
|
||||||
|
"loss": 1.568,
|
||||||
|
"step": 3500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9881965413121054,
|
||||||
|
"grad_norm": 1.2685039043426514,
|
||||||
|
"learning_rate": 6.176228383200659e-07,
|
||||||
|
"loss": 1.5983,
|
||||||
|
"step": 3600
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 100,
|
||||||
|
"max_steps": 3643,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 1,
|
||||||
|
"save_steps": 3643,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 1.1045534257958093e+17,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a4a252b43f42ad4d12be07cca1972ae8f64ffeeb2711dd370328e2c07e87a23c
|
||||||
|
size 5048
|
||||||
127
txtai-rag.py
Normal file
127
txtai-rag.py
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
|
||||||
|
import os
|
||||||
|
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||||
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
from torchsummary import summary
|
||||||
|
from accelerate import dispatch_model, infer_auto_device_map
|
||||||
|
from txtai import Embeddings
|
||||||
|
from txtai.pipeline import LLM
|
||||||
|
#pip3 install git+https://github.com/neuml/txtai#egg=txtai[pipeline-llm]
|
||||||
|
|
||||||
|
# Wikipedia Embeddings Database
|
||||||
|
embeddings = Embeddings()
|
||||||
|
embeddings.load(provider="huggingface-hub", container="neuml/txtai-wikipedia")
|
||||||
|
|
||||||
|
#os.environ['OMP_NUM_THREADS'] = '6'
|
||||||
|
|
||||||
|
# Load model and tokenizer
|
||||||
|
model_path = "Josephgflowers/Qllama-.5B-RAG-1"#
|
||||||
|
# Define the device (CPU or GPU)
|
||||||
|
device = torch.device("cuda")
|
||||||
|
#device = torch.device("cpu")
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(model_path,ignore_mismatched_sizes=True,torch_dtype=torch.float16).to(device)
|
||||||
|
|
||||||
|
print(model)
|
||||||
|
total_params = sum(p.numel() for p in model.parameters())
|
||||||
|
print("Total number of parameters: ", total_params)
|
||||||
|
|
||||||
|
sequence_length = 4096 # or whatever your specific sequence length is
|
||||||
|
embedding_size = 2048 # as per your model's definition
|
||||||
|
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_path)
|
||||||
|
stop_token = 128247
|
||||||
|
#'</s>' 2
|
||||||
|
|
||||||
|
def chat_with_model(prompt_text, stop_token, model, tokenizer):
|
||||||
|
# Encode the prompt text
|
||||||
|
encoded_prompt = tokenizer.encode(prompt_text, add_special_tokens=False, return_tensors="pt").to(device)
|
||||||
|
|
||||||
|
# Generate response
|
||||||
|
output_sequences = model.generate(
|
||||||
|
input_ids=encoded_prompt,
|
||||||
|
#max_length=len(encoded_prompt[0]) + 256,
|
||||||
|
max_new_tokens=1024,
|
||||||
|
temperature=0.1,
|
||||||
|
repetition_penalty=1.2,
|
||||||
|
top_k=20,
|
||||||
|
top_p=0.9,
|
||||||
|
do_sample=True,
|
||||||
|
num_return_sequences=1,
|
||||||
|
eos_token_id=stop_token
|
||||||
|
)
|
||||||
|
|
||||||
|
# Decode the generated sequence
|
||||||
|
generated_sequence = output_sequences[0].tolist()
|
||||||
|
text = tokenizer.decode(generated_sequence, clean_up_tokenization_spaces=True)
|
||||||
|
response_text = text[len(prompt_text):].strip() # Extract only the response text
|
||||||
|
#response_text = response_text.replace("<s>","").replace("</s>","")
|
||||||
|
return response_text
|
||||||
|
|
||||||
|
# Initialize conversation history
|
||||||
|
|
||||||
|
conversation_history = ''
|
||||||
|
# Get user's preference for input mode and character name
|
||||||
|
input_mode = 'text' ##input("Enter 'text' for text input or 'speech' for speech input: ").lower()
|
||||||
|
character_name = '<|user|>' # input("Enter your character name (USER, JONAH, JOSEPH, KIMBERLY, etc.): ")
|
||||||
|
|
||||||
|
# Chat loop
|
||||||
|
num_chat = 1
|
||||||
|
while num_chat <= 20:
|
||||||
|
if input_mode == 'text':
|
||||||
|
question = input(f"{character_name}: ")
|
||||||
|
user_input = question # Get text input from user
|
||||||
|
context = "\n".join([x["text"] for x in embeddings.search(question)])
|
||||||
|
elif input_mode == 'speech':
|
||||||
|
print("Press Enter and then speak your input.")
|
||||||
|
input() # Wait for user to hit Enter
|
||||||
|
with sr.Microphone() as source:
|
||||||
|
print("Listening...")
|
||||||
|
audio_data = recognizer.listen(source, timeout=5, phrase_time_limit=10)
|
||||||
|
try:
|
||||||
|
user_input = recognizer.recognize_google(audio_data)
|
||||||
|
print(f"USER: {user_input}")
|
||||||
|
except sr.RequestError:
|
||||||
|
print("API request failed. Please try again.")
|
||||||
|
continue
|
||||||
|
except sr.UnknownValueError:
|
||||||
|
print("Could not understand the audio. Please try again.")
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
print("Invalid input mode. Please restart and choose a valid mode.")
|
||||||
|
break
|
||||||
|
#print('History: '+ conversation_history)
|
||||||
|
prompt_text = f"""
|
||||||
|
<s>
|
||||||
|
<|system|>
|
||||||
|
You will be given text as context to answer a users question.
|
||||||
|
</s>
|
||||||
|
<|data|>
|
||||||
|
{context}
|
||||||
|
</s>
|
||||||
|
<|user|>
|
||||||
|
{question}
|
||||||
|
</s>
|
||||||
|
<|assistant|>
|
||||||
|
"""
|
||||||
|
response_text = chat_with_model(prompt_text, stop_token, model, tokenizer)
|
||||||
|
response_text = response_text.replace('<s>','')
|
||||||
|
|
||||||
|
|
||||||
|
# Extract assistant's response from the response_text
|
||||||
|
response_text = response_text.split('</s>\n', 1)[0] # Extract the first message from the assistant
|
||||||
|
|
||||||
|
print(f"Assistant: {response_text}","\n")
|
||||||
|
|
||||||
|
# Update conversation history
|
||||||
|
conversation_history += f"<s>\n<|system|>\nYou will are a helpful assistant.</s>\n{character_name}\n{user_input}\n</s>\n<|assistant|>\n {response_text}</s>\n\n"
|
||||||
|
if len(conversation_history) > 3092:
|
||||||
|
conversation_history = conversation_history[2048:]
|
||||||
|
else:
|
||||||
|
conversation_history = conversation_history
|
||||||
|
|
||||||
|
num_chat += 1
|
||||||
|
|
||||||
|
|
||||||
|
#"You will receive a user question and context that should be used to answer the question."
|
||||||
|
|
||||||
BIN
vocab.json
(Stored with Git LFS)
Normal file
BIN
vocab.json
(Stored with Git LFS)
Normal file
Binary file not shown.
Reference in New Issue
Block a user