export https_proxy=http://127.0.0.1:7890 http_proxy=http://127.0.0.1:7890 all_proxy=socks5://127.0.0.1:7890
unset https_proxy http_proxy all_proxy
python data_prepare/hh_filter.py

# 现有的攻击样本
## 1. llava的官方语料攻击，eps 16: visual_llava_llama_v2__demo
## 2. minigpt4的官方语料攻击，eps 16: results_minigpt_constrained_16_demo

# llava 官方攻击脚本
python minigpt_visual_attack.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --n_iters 700 --constrained --eps 32 --alpha 1 --save_dir visual_constrained_eps_32

# 这两个是样例 给定样本进行推理（用于debug用）
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_dna.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/visual_constrained_eps_16/bad_prompt_temp_5000.bmp --output_file /data//mm-safety/dna/reproduced_attach_eps16_result.jsonl --sample_num 5
# 给定样本文件夹进行推理，--segment进行分割
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/adversarial_images --output_fold /data//mm-safety/dna/adversarial_images

# 推理官方的6张图片
CUDA_VISIBLE_DEVICES=0 nohup python minigpt_inference_dna_one_image.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/clean.jpeg --output_fold /data//mm-safety/dna/adversarial_images --sample_num 5 > logs/official_clean.logs 2>&1 &
CUDA_VISIBLE_DEVICES=1 nohup python minigpt_inference_dna_one_image.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/prompt_constrained_16.bmp --output_fold /data//mm-safety/dna/adversarial_images --sample_num 5 > logs/official_eps16.logs 2>&1 &
CUDA_VISIBLE_DEVICES=2 python minigpt_inference_dna_one_image.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/prompt_constrained_32.bmp --output_fold /data//mm-safety/dna/adversarial_images --sample_num 5 > logs/official_eps32.logs 2>&1 &
CUDA_VISIBLE_DEVICES=3 python minigpt_inference_dna_one_image.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/prompt_constrained_64.bmp --output_fold /data//mm-safety/dna/adversarial_images --sample_num 5 > logs/official_eps64.logs 2>&1 &
CUDA_VISIBLE_DEVICES=4 python minigpt_inference_dna_one_image.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/adversarial_images/prompt_unconstrained.bmp --output_fold /data//mm-safety/dna/adversarial_images --sample_num 5 > logs/official_unconstrained.logs 2>&1 &
# 推理自己rlhf训练的图片
CUDA_VISIBLE_DEVICES=0 python minigpt_inference_dna_one_image.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/visual_constrained_eps_32_hh_rlhf/000000000139.bmp --output_fold /data//mm-safety/dna/visual_constrained_eps_32_hh_rlhf --sample_num 5
# 推理对应的原始图片
CUDA_VISIBLE_DEVICES=0 python minigpt_inference_dna_one_image.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_file /data//mm-safety/val2017/000000000139.jpg --output_fold /data//mm-safety/dna/val2017 --sample_num 5
# 8卡分segment推理
CUDA_VISIBLE_DEVICES=0 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 9 --sample_num 5 > /dev/null 2>&1 &
CUDA_VISIBLE_DEVICES=1 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 10 --sample_num 5 > /dev/null 2>&1 &
CUDA_VISIBLE_DEVICES=2 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 11 --sample_num 5 > /dev/null 2>&1 &
CUDA_VISIBLE_DEVICES=3 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 12 --sample_num 5 > /dev/null 2>&1 &
CUDA_VISIBLE_DEVICES=4 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 13 --sample_num 5 > /dev/null 2>&1 &
CUDA_VISIBLE_DEVICES=5 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 14 --sample_num 5 > /dev/null 2>&1 &
CUDA_VISIBLE_DEVICES=6 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 15 --sample_num 5 > /dev/null 2>&1 &
CUDA_VISIBLE_DEVICES=7 nohup python minigpt_inference_dna_fold.py --cfg-path eval_configs/minigpt4_eval.yaml  --gpu-id 0 --image_fold /data//mm-safety/val2017 --output_fold /data//mm-safety/dna/val2017 --all_segments 24 --segment_id 16 --sample_num 5 > /dev/null 2>&1 &

# openai_evaluation
python openai_evaluation.py --response_file /data//mm-safety/dna/adversarial_images/clean.jsonl
python openai_evaluation.py --response_file /data//mm-safety/dna/adversarial_images/prompt_constrained_16.jsonl
python openai_evaluation.py --response_file /data//mm-safety/dna/adversarial_images/prompt_constrained_32.jsonl
python openai_evaluation.py --response_file /data//mm-safety/dna/adversarial_images/prompt_constrained_64.jsonl
python openai_evaluation.py --response_file /data//mm-safety/dna/adversarial_images/prompt_unconstrained.jsonl
python openai_evaluation.py --response_file /data//mm-safety/dna/val2017/000000000139.jsonl
python openai_evaluation.py --response_fold /data//mm-safety/dna/val2017 --result_fold /data//mm-safety/openai_evaluation/dna/val2017
# evaluation summary
python openai_evaluation_summarize.py --evaluation_file /data//mm-safety/dna/adversarial_images/clean_openai_evaluation.jsonl
python openai_evaluation_summarize.py --evaluation_file /data//mm-safety/dna/adversarial_images/prompt_constrained_16_openai_evaluation.jsonl
python openai_evaluation_summarize.py --evaluation_file /data//mm-safety/dna/adversarial_images/prompt_constrained_32_openai_evaluation.jsonl
python openai_evaluation_summarize.py --evaluation_file /data//mm-safety/dna/adversarial_images/prompt_constrained_64_openai_evaluation.jsonl
python openai_evaluation_summarize.py --evaluation_file /data//mm-safety/dna/adversarial_images/prompt_unconstrained_openai_evaluation.jsonl
python openai_evaluation_summarize.py --evaluation_file /data//mm-safety/dna/visual_constrained_eps_32_hh_rlhf/000000000139_openai_evaluation.jsonl
python openai_evaluation_summarize.py --evaluation_file /data//mm-safety/dna/val2017/000000000139_openai_evaluation.jsonl
python openai_evaluation_summarize.py --result_fold /data//mm-safety/openai_evaluation/dna/val2017


# 准备图像文本对
python embedding_analysis/prepare_img_text_pair.py --result_fold /data//mm-safety/openai_evaluation/dna/val2017 --harmful_save_file /data//mm-safety/harmful_detection/raw_harmful_img_text_pair.jsonl --safe_save_file /data//mm-safety/harmful_detection/raw_safe_img_text_pair.jsonl
# 准备图像文本对输入minigpt4后，最后一个token在minigpt4所有transformer层的输出
python embedding_analysis/minigpt_img_text_emb_inference_train.py --img_text_pair_path /data//mm-safety/harmful_detection/raw_harmful_img_text_pair.jsonl

# vanilla_classifier
python vanilla_classifier/train.py --epochs 10 --train_emb_fold embedding_analysis/llava_hh_train_embedding --test_emb_fold embedding_analysis/llava_dc_test_demo_32_embedding

CUDA_VISIBLE_DEVICES=0 python embedding_analysis/minigpt_img_emb_inference_train.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --attacked_image_fold visual_constrained_eps_32_hh_rlhf --raw_image_fold val2017 --output_fold vanilla_classifier/minigpt4_train_emb

CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/minigpt4_add_random_noise.py --gpu_id 0 --attacked_image_fold visual_constrained_minpgpt_test --raw_image_fold test2017 --output_fold visual_constrained_minpgpt_test_add_randomn_noise

CUDA_VISIBLE_DEVICES=0 python embedding_analysis/minigpt_img_emb_inference_test.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --attacked_image_fold visual_constrained_minpgpt_test --raw_image_fold test2017 --output_fold vanilla_classifier/minigpt4_test_emb --random_image_fold visual_constrained_minpgpt_test_add_randomn_noise

CUDA_VISIBLE_DEVICES=0 python embedding_analysis/minigpt_img_emb_inference_train.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --attacked_image_fold visual_constrained_eps_32_hh_rlhf --raw_image_fold val2017 --output_fold vanilla_classifier/minigpt4_train_emb

CUDA_VISIBLE_DEVICES=0 python embedding_analysis/minigpt_img_emb_inference_train.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --attacked_image_fold visual_constrained_llava_train --raw_image_fold val2017 --output_fold vanilla_classifier/llava_train_emb

CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd) python embedding_analysis/minigpt4_add_random_noise.py --gpu_id 0 --attacked_image_fold visual_constrained_llava_test --raw_image_fold test2017 --output_fold visual_constrained_llava_test_test_add_randomn_noise

CUDA_VISIBLE_DEVICES=0 python embedding_analysis/minigpt_img_emb_inference_test.py --cfg_path eval_configs/minigpt4_eval.yaml  --cfg_path eval_configs/minigpt4_eval.yaml --gpu_id 0 --attacked_image_fold visual_constrained_llava_test --raw_image_fold test2017 --output_fold vanilla_classifier/llava_test_emb --random_image_fold visual_constrained_llava_test_test_add_randomn_noise

python vanilla_classifier/train.py --epochs 10 --train_emb_fold /data//mm-safety/hidden_analysis/llava_hh_train_hidden_f_pca_x --test_emb_fold /data//mm-safety/hidden_analysis/minigpt_hh_test_hidden_f_pca
