1 line
919 KiB
Plaintext
1 line
919 KiB
Plaintext
|
{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LLM Abliteration v1.2 script, adapted for internlm/internlm2_5-7b-chat\n\nAuthor: byroneverson\n\nThis script ran at kaggle.com, accelerator: P100, persistence: Files only","metadata":{}},{"cell_type":"markdown","source":"# Download original abliterator script for harmful and harmless instructions txt files\nCredit: https://github.com/Sumandora/remove-refusals-with-transformers","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n!git clone https://github.com/Sumandora/remove-refusals-with-transformers.git","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install requirements for the entire process","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/remove-refusals-with-transformers\n!pip install -r requirements.txt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Download internlm/internlm2_5-7b-chat model locally","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n\nfrom huggingface_hub import snapshot_download\nsnapshot_download(repo_id=\"internlm/internlm2_5-7b-chat\", local_dir=\"./internlm2_5-7b-chat\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate layer embedding outputs for all sample instructions and save to disk","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n\nimport jaxtyping\nimport random\nimport os\nimport gc\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer, BitsAndBytesConfig\nimport einops\nfrom tqdm import tqdm\n\n# Clear memory of past model usage\nmodel = None\ntokenizer = None\ngc.collect()\ntorch.cuda.empty_cache()\n\ntorch.inference_mode()\n\nlocal_repo_dir = \"/kaggle/working/internlm2_5-7b-chat\"\nworking_dir = \"/kaggle/working\"\n\nmodel = AutoModelForCausalLM.from_pretrained(local_repo_dir, local_files_only=True, trust_remote_code=True, \n torch_dtype=torch.float16, \n quantization_config=BitsAndBytesConfig(load_in_4bit=True,\n bnb_4bit_compute_dtype=torch.float16))\ntokenizer = AutoTokenizer.from_pretrained(local_repo_dir, local_files_only=True, trust_remote_code=True)\n\n# Settings\n# I have used 128 and 256 with success but may as well use the max for a better estimation\ninstructions = 256 #512\n#layer_idx = int(len(model.model.layers) * 0.5) #6)\n\nprint(\"Instruction count: \" + str(instructions))\n\nwith open(working_dir + \"/remove-refusals-with-transformers/harmful.txt\", \"r\") as f:\n harmful = f.readlines()\n\nwith open(working_dir + \"/remove-refusals-with-transformers/harmless.txt\", \"r\") as f:\n harmless = f.readlines()\n\nharmful_instructions = random.sample(harmful, instructions)\nharmless_instructions = random.sample(harmless, instructions)\n\nharmful = None\nharmless = None\ngc.collect()\n\n# Progress\nmax_its = instructions * 2\nbar = tqdm(total=max_its)\n\n# Generate target layer hidden state files for harmful and harmless features\ndef save_target_hidden_states(prompt, index, feature):\n bar.update(n=1)\n toks = tokenizer.apply_chat_template(conversation=[{\"role\": \"user\", \"content\": prompt}], add_generation_prompt=True,\n return_tensors=\"pt\")\
|