update

Author: Alankrit Srivastava

README.md

@@ -1,4 +1,4 @@
-#### Optimising Magento 2 with Varnish Cache, Redis and Nginx SSL termination on the Multi-container Architecture Using Docker-Compose tool.
+### Optimising Magento 2 with Varnish Cache, Redis and Nginx SSL termination on the Multi-container Architecture Using Docker-Compose tool.
 
 This repository corresponds to architecture setup as mentioned in blog https://cloudkul.com/blog/integrate-magento-2-varnish-cache-redis-server-ssl-termination-using-docker-compose/ .
 
@@ -30,46 +30,41 @@ Magento 2 works out of box with Varnish Cache and provides its own VCL file for
 
 Nginx servers as reverse proxy server that receives traffic on port 80 and 443 and then proxy pass it to listening port of Varnish Cache server. It is done to deploy a way to direct both HTTP and HTTPS traffic to Varnish cache server which in turn, if needed, forward it apache2 server.
 
-
 In this project, we are using:
 
-> Operating system: Ubuntu 14.04
+> Operating system: Ubuntu 16.04
 
 > Web Server: Apache2
 
-> Database Server: Mysql-server-5.6
+> Database Server: Mysql-server-5.7
+
+> Cache Server: Varnish 4.1
 
-> Cache Server: Varnish 3.0.
+> PHP version: PHP-7.1
 
 > Redis server: Redis 
 
 > SSL server: Nginx 1.10.1
 
-> PHP version: PHP-7.0
-
-> Magento 2.1.6
-
-To begin with, please install docker and docker-compose on your ubuntu 14.04 server. 
+To begin with, please install docker and docker-compose on your ubuntu server. 
 
 Then follow the following steps:
 
 1). Clone or download this repository as 
 
-> git clone https://github.com/webkul/magento2-varnish-redis-ssl-docker-compose.git
+> git clone https://github.com/webkul/magento2-varnish-redis-ssl-docker-compose.git.
 
-2) Set/modify mysql root credentials. Go to ~/magento2-varnish-redis-ssl-docker-compose/docker-compose.yml and change mysql root password in database_server in:
+2) Set mysql root credentials and name of the database to be created in *database_server* block ~/magento2-varnish-redis-ssl-docker-compose/docker-compose.yml:
 
-> mysql_password=rootpassword123
+> mysql_password=
 
-3). Modify change database name, database user and mysql root credentials in ~/magento2-varnish-redis-ssl-docker-compose/database_server/mysql.sh in:
+> mysql_database=
 
-> database_name=magento_db                  ## Mention database name
+3). Download Magento 2 version you wish to dockerize and upload it in directory magento2 in parallel docker-compose.yml.
 
-> database_user=magento_user                ## Mention database user
+> Go to https://magento.com/tech-resources/download? .
 
-> database_root_password=rootpassword123    ## Mention mysql root password.
-
-4). Replace the IP address assigned to 'server_name' in ~/magento2-varnish-redis-ssl-docker-compose/ssl_server/default with your domain name or IP address.
+4). Replace localhost in 'server_name' in ~/magento2-varnish-redis-ssl-docker-compose/ssl_server/default with your domain name or IP address.
 
 5). Build the docker image.
 
@@ -89,18 +84,11 @@ Then follow the following steps:
 
 > docker ps
 
-9). Your database credentials are mentioned in mysql.sh file. Database user password will be randomly generated. Database user password will be stored in /var/log/check.log. To get database user password,
-
-> docker exec -ti mysql bash
- 
-> cat /var/log/check.log
- 
-or,
-
-> docker exec -i mysql cat /var/log/check.log
 
 Now, your server setup is all ready, now hit your domain name or IP to install Magento 2. Now to configure Varnish for Magento 2 and test its working, please refer to blog https://cloudkul.com/blog/magento-2-and-varnish-cache-integration-with-docker-compose/.
 
+> Use Database host as mysql (name or id of the mysql container).
+
 To configure Magento 2 for redis-server, please refer to blog https://cloudkul.com/blog/integrate-magento-2-varnish-cache-redis-server-ssl-termination-using-docker-compose/ .
 
 
@@ -109,3 +97,9 @@ To configure Magento 2 for redis-server, please refer to blog https://cloudkul.c
 Although we had secured our application code keeping it on our host but database is as important as server code. So in order to keep their backup we schedule a shell script that will take backups of all the databases present in mysql-server container and keep them in archived from on our host. Shell script is present on ~/magento2-varnish-redis-ssl-docker-compose/backups/db_backup.sh. Please refer to blog https://cloudkul.com/blog/integrate-magento-2-varnish-cache-redis-server-ssl-termination-using-docker-compose/  for backup management.
 
 If you face any issues, kindly report back.
+
+
+#### GETTING SUPPORT
+
+If you have any issues, contact us at [email protected] or raise ticket at https://webkul.uvdesk.com/
+ 

backups/backup.sh

@@ -1,17 +1,18 @@
 #!/bin/bash
 
-set -x
+set -u
 ## Mention your database container name
 container_name=mysql
 
-## Mention your mysql root password
-mysql_root_password=rootpassword123
+## Mention mysql root password
+
+MYSQL_ROOT_PASSWORD=mention_your_mysql_root_password
 
 DATE=`date +%F-%H-%M-%S`
 
-for database in `echo 'show databases;' | docker exec -i mysql mysql --user=root --password=$mysql_root_password | grep -v Database | grep -v information_schema | grep -v mysql | grep -v performance_schema`
+for database in `echo 'show databases;' | docker exec -i mysql mysql --user=root --password=$MYSQL_ROOT_PASSWORD | grep -v Database | grep -v information_schema | grep -v mysql | grep -v performance_schema`
 do
 echo $database
-docker exec $container_name mysqldump -u root -p$mysql_root_password $database > $database-$DATE.sql && tar -zcvf $database-$DATE.tar.gz $database-$DATE.sql && rm $database-$DATE.sql && echo "$database-$DATE.tar.gz has been created on `date`" >> database_backup.log
+docker exec $container_name mysqldump -u root -p$MYSQL_ROOT_PASSWORD $database > $database-$DATE.sql && tar -zcvf $database-$DATE.tar.gz $database-$DATE.sql && rm $database-$DATE.sql && echo "$database-$DATE.tar.gz has been created on `date`" >> database_backup.log
 done
 

cache_server/Dockerfile

@@ -1,25 +1,16 @@
-From ubuntu:14.04
+From ubuntu:16.04
 
 MAINTAINER Alankrit Srivastava [email protected]
 
 ##update server
 
 RUN apt-get update \
-
 ##install supervisor and setup supervisord.conf file
-
 && apt-get install -y supervisor \
- 
 && mkdir -p /var/log/supervisor \
-
 ##install varnish
-
 && apt-get -y install varnish \
-
 && rm /etc/varnish/default.vcl \
-
 && rm /etc/default/varnish 
-
-EXPOSE 6082 80
-
+EXPOSE 6082 6081
 CMD ["/usr/bin/supervisord"]

cache_server/default.vcl

@@ -1,51 +1,53 @@
+
+vcl 4.0;
+
 import std;
-# The minimal Varnish version is 3.0.5
+# The minimal Varnish version is 5.0
 # For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'
 
-
 backend default {
     .host = "apache2";
     .port = "8080";
+    .first_byte_timeout = 600s;
 }
 
 acl purge {
-    "127.0.0.1";
+    "localhost";
 }
 
 sub vcl_recv {
-    if (req.restarts == 0) {
-        if (req.http.x-forwarded-for) {
-            set req.http.X-Forwarded-For =
-            req.http.X-Forwarded-For + ", " + client.ip;
-        } else {
-            set req.http.X-Forwarded-For = client.ip;
-        }
-    }
-
-    if (req.request == "PURGE") {
+    if (req.method == "PURGE") {
         if (client.ip !~ purge) {
-            error 405 "Method not allowed";
+            return (synth(405, "Method not allowed"));
+        }
+        # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
+        # has been added to the response in your backend server config. This is used, for example, by the
+        # capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
+        if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
+            return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
+        }
+        if (req.http.X-Magento-Tags-Pattern) {
+          ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
         }
-        if (!req.http.X-Magento-Tags-Pattern) {
-            error 400 "X-Magento-Tags-Pattern header required";
+        if (req.http.X-Pool) {
+          ban("obj.http.X-Pool ~ " + req.http.X-Pool);
         }
-        ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
-        error 200 "Purged";
+        return (synth(200, "Purged"));
     }
 
-    if (req.request != "GET" &&
-        req.request != "HEAD" &&
-        req.request != "PUT" &&
-        req.request != "POST" &&
-        req.request != "TRACE" &&
-        req.request != "OPTIONS" &&
-        req.request != "DELETE") {
+    if (req.method != "GET" &&
+        req.method != "HEAD" &&
+        req.method != "PUT" &&
+        req.method != "POST" &&
+        req.method != "TRACE" &&
+        req.method != "OPTIONS" &&
+        req.method != "DELETE") {
           /* Non-RFC2616 or CONNECT which is weird. */
           return (pipe);
     }
 
     # We only deal with GET and HEAD by default
-    if (req.request != "GET" && req.request != "HEAD") {
+    if (req.method != "GET" && req.method != "HEAD") {
         return (pass);
     }
 
@@ -54,73 +56,121 @@ sub vcl_recv {
         return (pass);
     }
 
+    # Bypass health check requests
+    if (req.url ~ "/pub/health_check.php") {
+        return (pass);
+    }
+
+    # Set initial grace period usage status
+    set req.http.grace = "none";
+
     # normalize url in case of leading HTTP scheme and domain
     set req.url = regsub(req.url, "^http[s]?://", "");
 
     # collect all cookies
     std.collect(req.http.Cookie);
 
-    # static files are always cacheable. remove SSL flag and cookie
-    if (req.url ~ "^/(pub/)?(media|static)/.*\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)$") {
-        unset req.http.Https;
-        unset req.http.X-Forwarded-Proto;
-        unset req.http.Cookie;
+    # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
+    if (req.http.Accept-Encoding) {
+        if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
+            # No point in compressing these
+            unset req.http.Accept-Encoding;
+        } elsif (req.http.Accept-Encoding ~ "gzip") {
+            set req.http.Accept-Encoding = "gzip";
+        } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
+            set req.http.Accept-Encoding = "deflate";
+        } else {
+            # unkown algorithm
+            unset req.http.Accept-Encoding;
+        }
     }
 
-    set req.grace = 1m;
+    # Remove Google gclid parameters to minimize the cache objects
+    set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
+    set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
+    set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
+
+    # Static files caching
+    if (req.url ~ "^/(pub/)?(media|static)/") {
+        # Static files should not be cached by default
+        return (pass);
+
+        # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
+        #unset req.http.Https;
+        #unset req.http.X-Forwarded-Proto;
+        #unset req.http.Cookie;
+    }
 
-    return (lookup);
+    return (hash);
 }
 
 sub vcl_hash {
     if (req.http.cookie ~ "X-Magento-Vary=") {
         hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
     }
 
+    # For multi site configurations to not cache each other's content
+    if (req.http.host) {
+        hash_data(req.http.host);
+    } else {
+        hash_data(server.ip);
+    }
+
+    # To make sure http users don't see ssl warning
     if (req.http.X-Forwarded-Proto) {
         hash_data(req.http.X-Forwarded-Proto);
     }
 
 }
 
-sub vcl_fetch {
+sub vcl_backend_response {
+
+    set beresp.grace = 3d;
+
     if (beresp.http.content-type ~ "text") {
         set beresp.do_esi = true;
     }
 
-    if (req.url ~ "\.js$" || beresp.http.content-type ~ "text") {
+    if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
         set beresp.do_gzip = true;
     }
 
     # cache only successfully responses and 404s
     if (beresp.status != 200 && beresp.status != 404) {
         set beresp.ttl = 0s;
-        return (hit_for_pass);
+        set beresp.uncacheable = true;
+        return (deliver);
     } elsif (beresp.http.Cache-Control ~ "private") {
-        return (hit_for_pass);
+        set beresp.uncacheable = true;
+        set beresp.ttl = 86400s;
+        return (deliver);
     }
 
     if (beresp.http.X-Magento-Debug) {
         set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
     }
 
     # validate if we need to cache it and prevent from setting cookie
-    # images, css and js are cacheable by default so we have to remove cookie also
-    if (beresp.ttl > 0s && (req.request == "GET" || req.request == "HEAD")) {
+    if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
         unset beresp.http.set-cookie;
-            if (req.url !~ "\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)(\?|$)") {
-            set beresp.http.Pragma = "no-cache";
-            set beresp.http.Expires = "-1";
-            set beresp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
-            set beresp.grace = 1m;
-        }
     }
+
+   # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
+   if (beresp.ttl <= 0s ||
+        beresp.http.Surrogate-control ~ "no-store" ||
+        (!beresp.http.Surrogate-Control && beresp.http.Vary == "*")) {
+        # Mark as Hit-For-Pass for the next 2 minutes
+        set beresp.ttl = 120s;
+        set beresp.uncacheable = true;
+    }
+    return (deliver);
 }
 
 sub vcl_deliver {
     if (resp.http.X-Magento-Debug) {
-        if (obj.hits > 0) {
+        if (resp.http.x-varnish ~ " ") {
             set resp.http.X-Magento-Cache-Debug = "HIT";
+            set resp.http.Grace = req.http.grace;
         } else {
             set resp.http.X-Magento-Cache-Debug = "MISS";
         }
@@ -136,3 +186,24 @@ sub vcl_deliver {
     unset resp.http.Via;
     unset resp.http.Link;
 }
+
+sub vcl_hit {
+    if (obj.ttl >= 0s) {
+        # Hit within TTL period
+        return (deliver);
+    }
+    if (std.healthy(req.backend_hint)) {
+        if (obj.ttl + 300s > 0s) {
+            # Hit after TTL expiration, but within grace period
+            set req.http.grace = "normal (healthy server)";
+            return (deliver);
+        } else {
+            # Hit after TTL and grace expiration
+            return (miss);
+        }
+    } else {
+        # server is not healthy, retrieve from cache
+        set req.http.grace = "unlimited (unhealthy server)";
+        return (deliver);
+    }
+}