nginx caching few upstreams on same server

I'm trying to test to build a nginx server to cache few servers. My nginx conf is like that :
=====================================================
...
http {

upstream srv1 {
ip_hash;
server srv1.domain1.fr:443;
}

upstream srv2 {
ip_hash;
server srv2.domain2.fr:443;
}
...
proxy_redirect off;
proxy_http_version 1.1;
proxy_read_timeout 10s;
proxy_send_timeout 10s;
proxy_connect_timeout 10s;
proxy_cache_path /nginx/cache/cache_temp use_temp_path=off keys_zone=cache_temp:10m max_size=10g inactive=10m;
proxy_cache cache_temp;
proxy_cache_methods GET HEAD;
proxy_cache_key $uri;
proxy_cache_valid 404 3s;
proxy_cache_lock on;
proxy_cache_lock_age 5s;
proxy_cache_lock_timeout 1h;
proxy_ignore_headers Cache-Control;
proxy_ignore_headers Set-Cookie;
proxy_cache_use_stale updating;
...
#srv1
server {
listen 443 ssl http2;
server_name srv1.domain1.fr;

all ssl settings...

location ~* \.(gif|jpg|jpeg|png|wmv|avi|mpg|mpeg|mp4|htm|html|js|css|mp3|swf|ico|flv|woff|woff2|ttf|svg)$ {
proxy_cache_valid 12h;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
add_header X-Cache $upstream_cache_status;
proxy_pass https://srv1;
}

location / {
proxy_cache_valid 12h;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
add_header X-Cache $upstream_cache_status;
proxy_pass https://srv1;
}
}

#srv2
server {
listen 443 ssl http2;
server_name srv2.domain2.fr;

all ssl settings...

location ~* \.(gif|jpg|jpeg|png|wmv|avi|mpg|mpeg|mp4|htm|html|js|css|mp3|swf|ico|flv|woff|woff2|ttf|svg)$ {
proxy_cache_valid 12h;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
add_header X-Cache $upstream_cache_status;
proxy_pass https://srv2;
}

location / {
proxy_cache_valid 12h;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
add_header X-Cache $upstream_cache_status;
proxy_pass https://srv2;
}
}
====================================================

so in my dns, I put the same IP for srv1.domain1.fr and srv2.domain2.fr that works well but when I switch between both, issue occured : cache is the same so I try to find a way to get separated cache

any idea ? thanks