export https_proxy=http://127.0.0.1:7890 http_proxy=http://127.0.0.1:7890 all_proxy=socks5://127.0.0.1:7890
unset https_proxy http_proxy all_proxy
python data_prepare/hh_filter.py

# 数据清理
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python data_prepare/hh_pretokenize.py
# one_for_all_attack （可以忽略）
CUDA_VISIBLE_DEVICES=0 nohup python minigpt_visual_attack.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --n_iters 100 --constrained --eps 32 --alpha 1 --save_dir visual_constrained_eps_32_ofaa --attack_text_path /data//mm-safety/harmful_corpus/one_for_all_attack.csv --batch_size 1 --segment_id 0 --all_segments 8 >/dev/null 2>&1 &

# image embedding analysis
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) CUDA_VISIBLE_DEVICES=0 python embedding_analysis/add_random_noise.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --attacked_image_fold visual_constrained_eps_32_hh_rlhf --raw_image_fold val2017 --output_fold visual_constrained_eps_32_hh_rlhf_random_noise
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) CUDA_VISIBLE_DEVICES=0 python embedding_analysis/minigpt_img_emb_inference.py --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --attacked_image_fold visual_constrained_eps_32_hh_rlhf --random_image_fold visual_constrained_eps_32_hh_rlhf_random_noise --raw_image_fold val2017 --output_fold visual_constrained_eps_32_hh_rlhf_embedding --batch_size 1
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/emb_clustering.py

# attacked output generation
CUDA_VISIBLE_DEVICES=0 python minigpt_inference_dna.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/visual_constrained_eps_32_hh_rlhf --output_fold /data//mm-safety/visual_constrained_eps_32_hh_rlhf_responses --sample_num 5

# minigpt4 攻击脚本
CUDA_VISIBLE_DEVICES=0 nohup python minigpt_visual_attack_hh_rlhf.py --cfg_path eval_configs/minigpt4_eval.yaml --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --n_iter 700 --constrained --eps 32 --alpha 1 --save_dir visual_constrained_eps_32_hh_rlhf --batch_size 16 --segment_id 0 --all_segments 8 >/dev/null 2>&1 &

# evaluation 1
CUDA_VISIBLE_DEVICES=0 python minigpt_inference_dna.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/visual_constrained_eps_32_hh_rlhf/000000000139.bmp --output_file /data//mm-safety/visual_constrained_eps_32_hh_rlhf/result.jsonl --sample_num 5

CUDA_VISIBLE_DEVICES=1 python minigpt_visual_attack_hh_rlhf.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --n_iters 5000 --constrained --eps 16 --alpha 1 --save_dir visual_constrained_eps_16_hh_rlhf_img --train_data_file /data//mm-safety/data_prepare/hh_harmless/train_filtered.jsonl
CUDA_VISIBLE_DEVICES=1 python minigpt_visual_attack.py --cfg_path eval_configs/minigpt4_eval.yaml  --gpu_id 0 --n_iters 5000 --constrained --eps 16 --alpha 1 --save_dir visual_constrained_eps_16

# evaluation 2
CUDA_VISIBLE_DEVICES=2 python minigpt_test_manual_prompts_visual_llm.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_path /data//mm-safety/visual_constrained_eps_16_hh_rlhf/bad_prompt_temp_400.bmp
CUDA_VISIBLE_DEVICES=3 python minigpt_test_manual_prompts_visual_llm.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_path /data//mm-safety/visual_constrained_eps_16/bad_prompt_temp_400.bmp
CUDA_VISIBLE_DEVICES=4 python minigpt_test_manual_prompts_visual_llm.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_path /data//mm-safety/adversarial_images/prompt_constrained_16.bmp

# evaluation 3
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_dna.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/prompt_unconstrained.bmp --output_file /data//mm-safety/dna/base_attach_eps16_result.jsonl --sample_num 5
CUDA_VISIBLE_DEVICES=3 python minigpt_inference_dna.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/clean.jpeg --output_file /data//mm-safety/dna/clearn_result.jsonl --sample_num 5
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_dna.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/visual_constrained_eps_16/bad_prompt_temp_5000.bmp --output_file /data//mm-safety/dna/reproduced_attach_eps16_result.jsonl --sample_num 5

# evaluation 4
CUDA_VISIBLE_DEVICES=3 python minigpt_inference_hh_rlhf.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/visual_constrained_eps_16/bad_prompt_temp_5000.bmp --output_file /data//mm-safety/visual_constrained_eps_16/result.jsonl --sample_num 10
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_hh_rlhf.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/clean.jpeg --output_file /data//mm-safety/visual_constrained_eps_16_hh_rlhf_img/clean_ab.jsonl --sample_num 10
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_hh_rlhf.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/prompt_constrained_16.bmp --output_file /data//mm-safety/visual_constrained_eps_16_hh_rlhf_img/adversarial_attack_ab.jsonl --sample_num 10
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_hh_rlhf.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/clean.jpeg --output_file /data//mm-safety/visual_constrained_eps_16_hh_rlhf_img/random_ab.jsonl --random_attack

# evaluation 5
python get_metric.py --input /data//mm-safety/visual_constrained_eps_16_hh_rlhf/result.jsonl --output /data//mm-safety/visual_constrained_eps_16_hh_rlhf/result_eval.jsonl
python cal_metrics.py --input /data//mm-safety/visual_constrained_eps_16_hh_rlhf/result_eval.jsonl
CUDA_VISIBLE_DEVICES=3 python minigpt_inference.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/visual_constrained_eps_16/bad_prompt_temp_5000.bmp --output_file /data//mm-safety/visual_constrained_eps_16/result.jsonl
python get_metric.py --input /data//mm-safety/visual_constrained_eps_16/result.jsonl --output /data//mm-safety/visual_constrained_eps_16/result_eval.jsonl
python cal_metrics.py --input /data//mm-safety/visual_constrained_eps_16/result_eval.jsonl
CUDA_VISIBLE_DEVICES=4 python minigpt_inference.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/prompt_constrained_16.bmp --output_file /data//mm-safety/result.jsonl
python get_metric.py --input /data//mm-safety/result.jsonl --output /data//mm-safety/result_eval.jsonl
python cal_metrics.py --input /data//mm-safety/result_eval.jsonl

## llava
### add_noise
CUDA_VISIBLE_DEVICES=6 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/llava_add_random_noise.py --gpu_id 0 --attacked_image_fold visual_llava_llama_v2_demo --raw_image_fold test2017 --output_fold visual_llava_llama_v2_demo_add_randomn_noise
### get_embedding
#### train
CUDA_VISIBLE_DEVICES=6 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/llava_llama_v2_img_emb_inference_train.py --gpu_id 0 --attacked_image_fold visual_constrained_llava_train --raw_image_fold val2017 --output_fold llava_train_embedding --batch_size 1
#### test
CUDA_VISIBLE_DEVICES=4 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/llava_llama_v2_img_emb_inference_test.py --gpu_id 0 --attacked_image_fold visual_llava_llama_v2_demo --raw_image_fold test2017 --random_image_fold visual_llava_llama_v2_demo_add_randomn_noise --output_fold visual_llava_llama_v2_demo_embeding --batch_size 1
### get_direction
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/get_direction.py --data_dir llava_train_embedding --save_file embedding_analysis/llava_direction.pkl
### evaluation
#### visual_llava_llama_v2_demo_embeding
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/test_direction_1.py --direction_file embedding_analysis/llava_direction.pkl --test_fold visual_llava_llama_v2_demo_embeding
#### llava_test_embedding
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/test_direction_1.py --direction_file embedding_analysis/llava_direction.pkl --test_fold llava_test_embedding

## minigpt4
### add_noise
CUDA_VISIBLE_DEVICES=6 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/minigpt4_add_random_noise.py --gpu_id 0 --attacked_image_fold results_minigpt_constrained_16_demo --raw_image_fold test2017 --output_fold results_minigpt_constrained_16_demo_add_randomn_noise
### get_embedding
#### train
CUDA_VISIBLE_DEVICES=6 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/minigpt_img_emb_inference_train.py --gpu_id 0 --attacked_image_fold visual_constrained_eps_32_hh_rlhf --raw_image_fold val2017 --output_fold minigpt_train_embedding --batch_size 1
#### test
CUDA_VISIBLE_DEVICES=4 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/minigpt_img_emb_inference_test.py --gpu_id 0 --attacked_image_fold results_minigpt_constrained_16_demo --raw_image_fold test2017 --random_image_fold results_minigpt_constrained_16_demo_add_randomn_noise --output_fold results_minigpt_constrained_16_demo_embedding --batch_size 1
### get_direction
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/get_direction.py --data_dir minigpt_train_embedding --save_file embedding_analysis/minigpt_direction.pkl
### evaluation
#### results_minigpt_constrained_16_demo_embedding
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/test_direction_1.py --direction_file embedding_analysis/minigpt_direction.pkl --test_fold results_minigpt_constrained_16_demo_embedding
#### minigpt_test_embedding
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/test_direction_1.py --direction_file embedding_analysis/minigpt_direction.pkl --test_fold minigpt_test_embedding

CUDA_VISIBLE_DEVICES=6 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/minigpt_img_emb_inference_train.py --gpu_id 0 --attacked_image_fold visual_constrained_eps_32_hh_rlhf --raw_image_fold val2017 --output_fold minigpt_train_embedding --batch_size 1
CUDA_VISIBLE_DEVICES=6 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/minigpt_img_emb_inference_test.py --gpu_id 0 --attacked_image_fold visual_constrained_minpgpt_test --raw_image_fold test2017 --random_image_fold visual_constrained_test_minigpt4_add_random_noise --output_fold minigpt_test_embedding --batch_size 1
CUDA_VISIBLE_DEVICES=4 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/llava_llama_v2_img_emb_inference_test.py --gpu_id 0 --attacked_image_fold visual_llava_llama_v2_demo --raw_image_fold test2017 --random_image_fold visual_constrained_test_llava_add_random_noise --output_fold llava_demo_test_embedding --batch_size 1

PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/get_direction.py --data_dir llava_train_embedding --save_file embedding_analysis/llava_direction.pkl
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/get_direction.py --data_dir minigpt_train_embedding --save_file embedding_analysis/minigpt_direction.pkl
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/test_direction_1.py --direction_file embedding_analysis/llava_direction.pkl --test_fold llava_test_embedding
PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/test_direction_1.py --direction_file embedding_analysis/minigpt_direction.pkl --test_fold minigpt_test_embedding