diff --git a/2016-10-18-Basic-Git-Commands.md b/2016-10-18-Basic-Git-Commands.md new file mode 100644 index 0000000000000..0b541e804da74 --- /dev/null +++ b/2016-10-18-Basic-Git-Commands.md @@ -0,0 +1,91 @@ +--- +layout: post +title: Basic Git Commands +comment: true +--- + +In this post, I talk about some of the most common git commands that I use. It is my version of the Git cheat sheet. + +## Creating a repository + +Initializes an empty repository in the current directory. + + git init + +## Adding files + +Adding the files, either one at a time or all the files + + git add * + git add file-name + +## Current status of Git + +Details on current repository, whether any files needs to be checked in, etc. + + git status + +## Committing changes + + git commit -m "message" + +Listing all changes to repository + + git log + +## Branching information + +### List details of the current repository branch. + + git branch -a + +### Create a new branch +This will create a new branch. If a branch already exists, it simply throws an error. + + git checkout -b development + +### Switch branch + + git checkout development + +### Merge a branch +Assuming the user wants to merge "development" to "master", this will work. The __"--no-ff"__ option will + + git merge development --no-ff + +## Working with tags +### Listing the tags +Lists all the existing tags + + git tag + +### Creating a simple tag +This will create a tag but have no information about it. + + git tag tag1 + +### Creating an annotated tag +This will add more info when creating the tag. + + git tag -a v1 -m "Version 1.0 release" + +### Show the details of a tag +This will show the last committed details and the files that are present in a tag. + + git show tag1 + +## Git Pull +Git pull is a bit more complex command. It is used when there are multiple developers but one authorized person responsible to merge the code. In this case, all the develpers make their change and create a __pull__ request. It is like the developer telling the commitor to review and merge the change if it passes the acceptance criteria. + +For git pull to work, you will need to do the following: +* Create a branch +* Make changes to the branch +* Add and commit the changes +* Push the change to the remote repository +* Now, request a git pull on the branch name + +Suppose the branch created is bug-fix-2001 the command would be: + + git pull bug-fix-2001 + +The code reviewer will be notified and they can take further action. \ No newline at end of file diff --git a/CNAME b/CNAME deleted file mode 100644 index 8b137891791fe..0000000000000 --- a/CNAME +++ /dev/null @@ -1 +0,0 @@ - diff --git a/_config.yml b/_config.yml index d4916414195c9..97e568f4cdbdd 100644 --- a/_config.yml +++ b/_config.yml @@ -3,14 +3,27 @@ # # Name of your site (displayed in the header) -name: Your Name +name: Akshay Ranganath's Blogs +author: rakshay + + +#adding twitter details +twitter: + site: Akshay Ranganath's Blogs + username: rakshay # Short bio or description (displayed in the header) -description: Web Developer from Somewhere +description: Blogs about Media Optimization, Web Performance, SEO and web technology. +tagline: Musings from a Tech Entrepreneur + +paginate: 8 +paginate_path: '/blog/page:num' # URL of your avatar or profile pic (you could use your GitHub profile pic) -avatar: https://raw.githubusercontent.com/barryclark/jekyll-now/master/images/jekyll-logo.png +avatar: /images/blog/github_profile.jpg + +logo: /images/blog/github_profile.jpg # # Flags below are optional # @@ -21,12 +34,12 @@ footer-links: email: facebook: flickr: - github: barryclark/jekyll-now + github: akshayranganath instagram: - linkedin: + linkedin: akshayranganath pinterest: - rss: # just type anything here for a working RSS icon - twitter: jekyllrb + rss: rss # just type anything here for a working RSS icon + twitter: rakshay stackoverflow: # your stackoverflow profile, e.g. "users/50476/bart-kiers" youtube: # channel/ or user/ googleplus: # anything in your profile username that comes after plus.google.com/ @@ -34,14 +47,14 @@ footer-links: # Enter your Disqus shortname (not your username) to enable commenting on posts # You can find your shortname on the Settings page of your Disqus account -disqus: +# disqus: akshayranganath # Enter your Google Analytics web tracking code (e.g. UA-2110908-2) to activate tracking -google_analytics: +google_analytics: UA-92984102-1 # Your website URL (e.g. http://barryclark.github.io or http://www.barryclark.co) # Used for Sitemap.xml and your RSS feed -url: +url: https://akshayranganath.github.io # If you're hosting your site at a Project repository on GitHub pages # (http://yourusername.github.io/repository-name) @@ -70,12 +83,15 @@ kramdown: # Set the Sass partials directory, as we're using @imports sass: - style: :expanded # You might prefer to minify using :compressed + style: :compressed #:expanded # You might prefer to minify using :compressed # Use the following plug-ins gems: - jekyll-sitemap # Create a sitemap using the official Jekyll sitemap gem - jekyll-feed # Create an Atom feed using the official Jekyll feed gem + - jekyll-seo-tag # Add SEO related tags and JSON-LD + - jekyll-paginate # Adding pagination capability + - jekyll-redirect-from #redirect functionalit # Exclude these files from your production _site exclude: @@ -84,3 +100,28 @@ exclude: - LICENSE - README.md - CNAME + +# inspired by https://github.com/codinfox/codinfox-lanyon +# Adding category and tags support +categories: + type: liquid + path: /categories/ +tags: + type: liquid + path: /tags/ + +nav: + Home: + - '/' + Blog: + - '/blog/' + - { Categories: '/blog/categories/' } + - { Tags: '/blog/tags/' } + About: + - '/about/' + Github Project: + - 'http://akshayranganath.github.io/' + +# Where are tags and categories displayed +tag_page: '/blog/tags/' +category_page: '/blog/categories/' diff --git a/_includes/analytics.html b/_includes/analytics.html index 5f346aee620c1..141f5aed5c29f 100644 --- a/_includes/analytics.html +++ b/_includes/analytics.html @@ -1,16 +1,13 @@ {% if site.google_analytics %} + + {% endif %} diff --git a/_includes/disqus.html b/_includes/disqus.html index c4b4d2394f04a..8ded46c00c52d 100644 --- a/_includes/disqus.html +++ b/_includes/disqus.html @@ -1,17 +1,17 @@ {% if site.disqus %}
-
- - +
{% endif %} \ No newline at end of file diff --git a/_includes/meta.html b/_includes/meta.html index b8f4a3308354a..6b30748da8c83 100644 --- a/_includes/meta.html +++ b/_includes/meta.html @@ -2,17 +2,8 @@ - - {% if page.excerpt %} - - - {% else %} - - - {% endif %} - - - {% if page.title %} - - - {% endif %} \ No newline at end of file + + {% seo title=false %} + + + diff --git a/_layouts/default.html b/_layouts/default.html index b2939c0bc4483..6d650d01c861b 100644 --- a/_layouts/default.html +++ b/_layouts/default.html @@ -1,7 +1,7 @@ - + - {% if page.title %}{{ page.title }} – {% endif %}{{ site.name }} – {{ site.description }} + {% if page.title %}{{ page.title }} – {% endif %}{{ site.name }} {% include meta.html %} @@ -9,6 +9,7 @@ + @@ -44,8 +45,7 @@

{{ site.name }}

{% include svg-icons.html %} - - + {% include analytics.html %} diff --git a/_layouts/page.html b/_layouts/page.html index c8f0164980270..844fdfede497a 100644 --- a/_layouts/page.html +++ b/_layouts/page.html @@ -1,6 +1,17 @@ --- layout: default --- +
@@ -9,4 +20,4 @@

{{ page.title }}

{{ content }}
-
+ diff --git a/_layouts/post.html b/_layouts/post.html index d27c48066a71e..9ad67ab2b4fa0 100644 --- a/_layouts/post.html +++ b/_layouts/post.html @@ -1,17 +1,13 @@ --- layout: default --- -

{{ page.title }}

- -
- {{ content }} -
-
Written on {{ page.date | date: "%B %e, %Y" }}
- +
+ {{ content }} +
{% include disqus.html %}
diff --git a/_posts/2014-3-3-Hello-World.md b/_posts/2014-3-3-Hello-World.md deleted file mode 100644 index d4665b6d18e9e..0000000000000 --- a/_posts/2014-3-3-Hello-World.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: post -title: You're up and running! ---- - -Next you can update your site name, avatar and other options using the _config.yml file in the root of your repository (shown below). - -![_config.yml]({{ site.baseurl }}/images/config.png) - -The easiest way to make your first post is to edit this one. Go into /_posts/ and update the Hello World markdown file. For more instructions head over to the [Jekyll Now repository](https://github.com/barryclark/jekyll-now) on GitHub. \ No newline at end of file diff --git a/_posts/2016-10-18-Basic-Git-Commands.md b/_posts/2016-10-18-Basic-Git-Commands.md new file mode 100644 index 0000000000000..0b541e804da74 --- /dev/null +++ b/_posts/2016-10-18-Basic-Git-Commands.md @@ -0,0 +1,91 @@ +--- +layout: post +title: Basic Git Commands +comment: true +--- + +In this post, I talk about some of the most common git commands that I use. It is my version of the Git cheat sheet. + +## Creating a repository + +Initializes an empty repository in the current directory. + + git init + +## Adding files + +Adding the files, either one at a time or all the files + + git add * + git add file-name + +## Current status of Git + +Details on current repository, whether any files needs to be checked in, etc. + + git status + +## Committing changes + + git commit -m "message" + +Listing all changes to repository + + git log + +## Branching information + +### List details of the current repository branch. + + git branch -a + +### Create a new branch +This will create a new branch. If a branch already exists, it simply throws an error. + + git checkout -b development + +### Switch branch + + git checkout development + +### Merge a branch +Assuming the user wants to merge "development" to "master", this will work. The __"--no-ff"__ option will + + git merge development --no-ff + +## Working with tags +### Listing the tags +Lists all the existing tags + + git tag + +### Creating a simple tag +This will create a tag but have no information about it. + + git tag tag1 + +### Creating an annotated tag +This will add more info when creating the tag. + + git tag -a v1 -m "Version 1.0 release" + +### Show the details of a tag +This will show the last committed details and the files that are present in a tag. + + git show tag1 + +## Git Pull +Git pull is a bit more complex command. It is used when there are multiple developers but one authorized person responsible to merge the code. In this case, all the develpers make their change and create a __pull__ request. It is like the developer telling the commitor to review and merge the change if it passes the acceptance criteria. + +For git pull to work, you will need to do the following: +* Create a branch +* Make changes to the branch +* Add and commit the changes +* Push the change to the remote repository +* Now, request a git pull on the branch name + +Suppose the branch created is bug-fix-2001 the command would be: + + git pull bug-fix-2001 + +The code reviewer will be notified and they can take further action. \ No newline at end of file diff --git a/_posts/2016-10-24-DevOps-Journey-Setting-up-Git-server-and-Client.md b/_posts/2016-10-24-DevOps-Journey-Setting-up-Git-server-and-Client.md new file mode 100644 index 0000000000000..3bc42bc504c5b --- /dev/null +++ b/_posts/2016-10-24-DevOps-Journey-Setting-up-Git-server-and-Client.md @@ -0,0 +1,67 @@ +--- +layout: post +title: DevOps Journey - Settup up Git Server and SSH enabled client +comment: true +--- + +In this post, I describe the steps needed to setup a Git server and configuring it to accept the git push command. + +## Background + +Before embarking on my DevOps journey, I wanted to setup a Git server and client. I did not want to use the public Github or any free service. I wanted to do it the hard way to ensure that I knew the process and could relate to the pain that ops feel in setting a new system up! + +## Installing git as a server + +I started out by reading up the steps at this page: Git on the Server – Setting Up the Server. It was generally well laid out and well-referenced by others. + +However, as all things Linux, one of the steps failed. When I followed the process and tried to do push my code to the remote Git server, it failed. After a bit of research, I found out these are the steps that works. + +### my computer + $ sudo adduser git + $ su git + $ cd + $ mkdir .ssh && chmod 700 .ssh + $ touch .ssh/authorized_keys && chmod 600 .ssh/authorized_keys + + $ cd /opt/git + $ git init --bare myproject.git #this step differs from the reference page + Initialized empty Git repository in /opt/git/project.git/ + +### on John's computer + $ cd myproject + $ git init + $ git add . + $ git commit -m 'initial commit' + $ git remote add origin git@gitserver:/opt/git/project.git + $ git push origin master + + + $ git clone git@gitserver:/opt/git/project.git + $ cd project + $ vim README + $ git commit -am 'fix for the README file' + $ git push origin master + +It was basically one line “$ git init –bare myproject.git” that caused the difference. Without this step, the git reference files were being created a directory above the folder marked as repository. The code push would fail and I’d only see errors. Once I made this correction, the error went away and the I was able to continue using Git. + +## Tweaking / Checking git settings + +After having installed Git, here are some tips to check or update the git settings. The first set lists the current settings on Git and the rest of the commands is for tweaking the git settings. Git settings is divided into 3 sections as follows: + +System settings: applies to all users and all repositories. +Global settings: applies to this users for all the user's repositories. +Local settings: applies to this repository. +Settings at Sytem is over-ridden by Global; settings at Global level are over-ridden by Local settings. + +### print the current Git configuration + git config --list + +### set the editor for this user globally + git config --global core.editor=vim + +### set the git user name locally for this repository + git config --local user.name + +### create a git ignore rules globally and + create .gitignore + git config --global core.excludesfile ~/.gitignore_global \ No newline at end of file diff --git a/_posts/2016-11-4-quotes-from-book-the-phoenix-project.md b/_posts/2016-11-4-quotes-from-book-the-phoenix-project.md new file mode 100644 index 0000000000000..bde7676ba7d72 --- /dev/null +++ b/_posts/2016-11-4-quotes-from-book-the-phoenix-project.md @@ -0,0 +1,57 @@ +--- +layout: post +title: DevOps Quotes from book - The Phoenix Project +comment: true +description: The Phoenix Project book is a good introduction to the approach of moving from traditional ops to devOps. This is my favorite set of devops quotes from the book. +--- + +Over last 2 weeks, I read the book **The Phoenix Project: A Novel about IT, devops and helping business win**. It was an immensely readable book that explains the need, the role and concepts of DevOps but as a novel. It is a thriller of a story and a wonderful way to get introduced to the concept of DevOps. I'd highly recommend this book. + +![Phoneix project book cover](/images/blog/phoenix_project_cover.jpg) + +I took some notes and here are some of my favorite quotes from the book: + +__.. The plot is simple:__ First, you take an urgent date-driven project, where the shipment date cannot be delayed because of external commitments made to Wall Street or customers. Then you add a bunch of developers who use up all the time in the schedule, leaving no time for testing or operations deployment. And because no one is willing to slip the deployment date, everyone after Development has to take outrageous and unacceptable shortcuts to hit the date. + +I always liked that phrase in Saving Private Ryan: “There’s a chain of command: gripes go up, not down.” + +“Your job as VP of IT Operations is to ensure the fast, predictable, and uninterrupted flow of planned work that delivers value to the business while minimizing the impact and disruption of unplanned work, so you can provide stable, predictable, and secure IT service.” + +__The First Way__ helps us understand how to create fast flow of work as it moves from Development into IT Operations, because that’s what’s between the business and the customer. __The Second Way__ shows us how to shorten and amplify feedback loops, so we can fix quality at the source and avoid rework. And the __Third Way__ shows us how to create a culture that simultaneously fosters experimentation, learning from failure, and understanding that repetition and practice are the prerequisites to mastery. + +He continues, “What use is it having all these offshore developers building features if we aren’t getting to market any faster? We keep lengthening the deployment intervals, so that we can get more features deployed in each batch.” + +4 types of work: + +* business projects +* internal projects +* changes & maintenance +* unplanned work + +You’ve just described __‘technical debt’__ that is not being paid down. It comes from taking shortcuts, which may make sense in the short-term. But like financial debt, the compounding interest costs grow over time. If an organization doesn’t pay down its technical debt, every calorie in the organization can be spent just paying interest, in the form of unplanned work. + +"Unplanned work has another side effect. When you spend all your time firefighting, there’s little time or energy left for planning. When all you do is react, there’s not enough time to do the hard mental work of figuring out whether you can accept new work. So, more projects are crammed onto the plate, with fewer cycles available to each one, which means more bad multitasking, more escalations from poor code, which mean more shortcuts..." + +Allspaw taught us that Dev and Ops working together, along with QA and the business, are a super-tribe that can achieve amazing things. They also knew that until code is in production, no value is actually being generated, because it’s merely WIP stuck in the system. He kept reducing the batch size, enabling fast feature flow. In part, he did this by ensuring environments were always available when they were needed. He automated the build and deployment process, recognizing that infrastructure could be treated as code, just like the application that Development ships. That enabled him to create a one-step environment creation and deploy procedure.. + +__“If you can’t out-experiment and beat your competitors in time to market and agility, you are sunk. Features are always a gamble. If you’re lucky, ten percent will get the desired benefits. So the faster you can get those features to market and test them, the better off you’ll be. Incidentally, you also pay back the business faster for the use of capital, which means the business starts making money faster, too."__ + +“In ten years, I’m certain every COO worth their salt will have come from IT. Any COO who doesn’t intimately understand the IT systems that actually run the business is just an empty suit, relying on someone else to do their job.” + +“I’ve long believed that to effectively manage IT is not only a critical competency but a significant predictor of company performance,” he explains. + +Just how did code deployment become routine? Because developers are constantly getting fast feedback on their work: when they write code, automated unit, acceptance, and integration tests are constantly being run in production-like environments, giving us continual assurance that the code and environment will operate as designed, and that we are always in a deployable state. And when the code is deployed, pervasive production metrics demonstrate to everyone that it is working, and the customer is getting value. + +DevOps shows how we optimize the IT value stream, converting business needs into capabilities and services that provide value for our customers. + +The term __“devops”__ was originally coined by Patrick Debois and Andrew Shafer in 2008, and it entered common usage in 2009 in the Velocity Conference community with the famous “10+ Deploys Per Day: Dev and Ops Cooperation at Flickr” presentation, given by John Allspaw and Paul Hammond. + +DevOps has benefitted tremendously from the work the Agile Community has done, showing how small teams operating with high trust combined with small batch sizes and smaller, more frequent software releases can dramatically increase productivity of Development organizations. + +And of course, DevOps extends and builds upon the practices of __“infrastructure as code”__ pioneered by Dr. Mark Burgess, as well as continuous integration and continuous deployment (pioneered by Jez Humble and David Farley), which is a prerequisite to achieving fast deployment flow. + +**Myth: DevOps means NoOps** +DevOps is sometimes incorrectly interpreted to be NoOps (i.e., IT Operations is entirely eliminated). However, more precisely, DevOps will often put more responsibility on Development to do code deployments and maintain service levels. This merely means that Development is taking over many of the IT Operations and operations engineering functions. +In order to support fast lead times and enable developer productivity, DevOps does require many IT Operations tasks to become self-service. In other words, instead of Development opening up a work ticket and waiting for IT Operations to complete the work, many of these activities will be automated so that developers can do it themselves (e.g., get a production-like Dev environment, add a feature metric for production telemetry). + +The most obvious manifestation of the Toyota Kata is the two-week improvement cycle, in which every work center supervisor must improve something (anything!) every two weeks. To quote Mr. Rother, “The practice of kata is the act of practicing a pattern so it becomes second nature. In its day-to-day management, Toyota teaches a way of working—a kata—that has helped make it so successful over the last six decades.” \ No newline at end of file diff --git a/_posts/2016-12-1-Performance-Testing-with-Jenkins.md b/_posts/2016-12-1-Performance-Testing-with-Jenkins.md new file mode 100644 index 0000000000000..ef9b65022e634 --- /dev/null +++ b/_posts/2016-12-1-Performance-Testing-with-Jenkins.md @@ -0,0 +1,101 @@ +--- +layout: post +title: Performance Testing with Jenkins +comment: true +--- + +In this blog post, I try to outline how Jenkins can be leveraged to perform Synthetic testing by using command line tool called Siege for automating the tests. + +If you are looking to setup a synthetic testing with Jenkins, please read the blog about [Synthetic testing with WPT and Jenkins](https://akshayranganath.github.io/Synthetic-Testing-With-WPT-and-Jenkins/). + +## Why Jenkins? +Jenkins is an excellent CI tool that has some really cool capabilities. One of the features I like is the scheduled build. It is very similar to a cron job. The place where it shines well is in managing the master/slave node execution. With Jenkins, you could designate an instance as the master and then enable it to run the different commands / processes on the slave nodes. This is the feature that helps in setting up synthentic tests. + + +## What is the idea? +My idea is that for a DevOps oriented organization, Jenkins could be used to run either quick tests or a scheduled synthetic tests. The power comes in by leveraging the AWS geo-regions to perform a distributed tests for measuring the response times from various geographies. + + + +## How will it work? +Let’s assume that we are using Amazon EC2 instances. We could create EC2 instances across different geo. Depending on the kind of tests required, it could in each location offered by Amazon or it could be setup for tests in a particular geo. All these instances would be child nodes that will be enabled to execute the synthetic test from the Jenkins master node. + + +## What kind of a test can Jenkins run? +I am still new to the world of Jenkins but, based on my understanding following types of tests could be executed: + +* full suite regression test from a single node +* run a check against a set of URLs for synthetic monitoring +* run a check for geo-based content validation +* warm CDN cache across geographies, if using a CDN + + +## Simple Demo +To prove that this idea can work, I leveraged [Siege](https://www.joedog.org/siege-home/) testing tool to execute a 10s load test on a single URL. This test was scheduled to execute every 10 minutes. I also enabled the logging with siege so that the historical set of results would be available for analysis. + +I did not use a master / slave setup in this test. So the load test was running from the same instance as the Jenkins server. + +### Setting up the scheduled build + +I set the script to run every 10 minutes. Here’s what this looks like: + +![Jenkins - scheduled build setup](/images/blog/jenkins_scheduling_build.png) + + +### Build Script + +For the build script, I just set up the test to run seige for 10s with a concurrency of 1. The test executes against a single URL. + +![Jenkins - Siege Script](/images/blog/jenkins_siege_script.png) + +#### Sample Output + +Here’s a sample output from each test run. + + Started by timer + Building on master in workspace /var/lib/jenkins/workspace/mytests/CachePreWarm + [CachePreWarm] $ /bin/sh -xe /tmp/hudson5180622479104364552.sh + + siege -v -c 1 -t 5s -l http://localhost/login?from=%2F + ** SIEGE 3.0.5 + ** Preparing 1 concurrent users for battle. + The server is now under siege... + + Lifting the server siege...- done. + + Transactions: 9 hits + Availability: 100.00 % + Elapsed time: 4.93 secs + Data transferred: 0.02 MB + Response time: 0.00 secs + Transaction rate: 1.83 trans/sec + Throughput: 0.00 MB/sec + Concurrency: 0.01 + Successful transactions: 9 + Failed transactions: 0 + Longest transaction: 0.01 + Shortest transaction: 0.00 + + +For my later analysis, I used the _/var/log/siege.log_. It has a CSV seperated values for the parameters listed in the build output above. So I can consolidate the results for extracting patterns or troubleshooting issues. + + 2016-12-01 18:39:09, 7, 4.80, 0, 0.00, 1.46, 0.00, 0.01, 7, 0 + 2016-12-01 19:44:05, 10, 4.93, 0, 0.00, 2.03, 0.00, 0.01, 10, 0 + 2016-12-01 19:54:05, 7, 4.93, 0, 0.00, 1.42, 0.00, 0.01, 7, 0 + 2016-12-01 20:04:05, 7, 4.93, 0, 0.00, 1.42, 0.00, 0.01, 7, 0 + 2016-12-01 20:14:05, 6, 4.93, 0, 0.00, 1.22, 0.00, 0.00, 6, 0 + 2016-12-01 20:24:05, 10, 4.93, 0, 0.00, 2.03, 0.00, 0.01, 10, 0 + 2016-12-01 20:34:05, 12, 4.93, 0, 0.00, 2.43, 0.00, 0.01, 12, 0 + 2016-12-01 20:44:05, 9, 4.92, 0, 0.01, 1.83, 0.00, 0.01, 9, 0 + 2016-12-01 20:54:05, 6, 4.92, 0, 0.00, 1.22, 0.00, 0.01, 6, 0 + 2016-12-01 21:04:05, 10, 4.93, 0, 0.00, 2.03, 0.00, 0.01, 10, 0 + 2016-12-01 21:14:05, 10, 4.93, 0, 0.00, 2.03, 0.00, 0.01, 10, 0 + 2016-12-01 21:24:05, 5, 4.93, 0, 0.00, 1.01, 0.00, 0.00, 5, 0 + 2016-12-01 21:34:05, 6, 4.92, 0, 0.00, 1.22, 0.00, 0.01, 6, 0 + 2016-12-01 21:44:05, 13, 4.93, 0, 0.00, 2.64, 0.00, 0.01, 13, 0 + 2016-12-01 21:54:05, 13, 4.93, 0, 0.00, 2.64, 0.00, 0.01, 13, 0 + 2016-12-01 22:04:05, 19, 4.92, 0, 0.00, 3.86, 0.00, 0.01, 19, 0 + 2016-12-01 22:14:05, 13, 4.93, 0, 0.00, 2.64, 0.00, 0.01, 13, 0 + + +## Conclusion +Jenkins offers an excellent way to automate the running of synthetic tests. The tests could be run after a deployment or periodically. The script to be executed is configurable and it is easily possible to extend this test to use more complex tests like Selenium based checks for transactions. It’ll be interesting to see how this technology pans out. \ No newline at end of file diff --git a/_posts/2016-12-12-Useful-SEO-tags-part-1.md b/_posts/2016-12-12-Useful-SEO-tags-part-1.md new file mode 100644 index 0000000000000..8bbb45706eb90 --- /dev/null +++ b/_posts/2016-12-12-Useful-SEO-tags-part-1.md @@ -0,0 +1,68 @@ +--- +layout: post +title: Useful SEO Tags - Part 1 +comment: true +--- + +In this blog post, I analyze the distribution and the use cases for the different kinds of __link__ tags. Starting with _link rel="canonical"_, I dig into the distribution of the other attributes and use cases for the tag. + +## SEO Tags in the wild +[Rand Fishkin](https://twitter.com/randfish) published a [whiteboard Friday](https://moz.com/blog/category/whiteboard-friday) on the topic of [which tags matter for SEO](https://moz.com/blog/which-page-markup-tags-still-matter-for-seo-whiteboard-friday). While reading it, I was intrigued to check the kind of tags that are actually being used in the wild. + +So I thought of running my own research to see the kind of trends and usage of tags across the Alexa Top 1000 websites. Since I did not have access to the Alexa list, I used the top 1000 URLs from [HTTPArchive](http://httparchive.org/) instead. + +In this first part of the blog, I summarize my findings on the  tag. + +## Meta Tags +One of the first things I checked was the use of the tag. This was interesting to me after I learnt the foundational tag __link__ and a slightly less used but interesting use of __link canonical__ for geo targeting. + +For the top websites, I assumed that [link rel="canonical"](https://moz.com/learn/seo/canonicalization) would be quite relevant and [hreflang](https://moz.com/learn/seo/hreflang-tag) would play an important role in geo-targeting. However, my research did have a limitation in that I was accessing just the home page. So, the potential to see the canonical links would be fewer. + +I was correct in this assumption. Of the top 25 use of _link rel_, here's the distribution I saw: + +| Attribute | Count | +| --------- | ----- | +| alternate | 4046 | +| stylesheet | 2550 | +| dns-prefetch | 1269 | +| apple-touch-icon | 1026 | +| shortcut | 585 | +| icon | 528 | +| canonical | 414 | +| apple-touch-icon-precomposed | 305 | +| preconnect | 170 | +| search | 155 | +| publisher | 101 | +| manifest | 69 | +| mask-icon | 49 | +| image_src | 42 | +| chrome-webstore-item | 35 | +| preload | 33 | +| next | 30 | +| shortlink | 25 | +| edituri | 21 | +| wlwmanifest | 20 | +| prefetch | 13 | +| pingback | 12 | +| profile | 11 | +| author | 11 | +| apple-touch-startup-image | 8 | + + +I have uploaded the raw data here: [https://gist.github.com/akshayranganath/9e953a32c6ed8066f006301bc02bef1c](https://gist.github.com/akshayranganath/9e953a32c6ed8066f006301bc02bef1c). + +Here's the distribution again: + +![SEO Link Distribution](/images/blog/seo_link_distribution.png) + +When I looked at the <link> tags, I wasn't sure on the usage and had to do a bit of read-up. Here's a summary of some of the tags: + +* __rel="search":__ As a user, you can configure alternate search engines like "imdb" on browsers. This tag provides a mechanism to provide hint that your site has the ability for such a search functionality. The concept is called OpenSearch and you can read about it's use at Aaron Parecki's blog. And here's a sample OpenSearch file from Airbnb: https://www.airbnb.com/opensearch.xml. +* __rel="image_src"__: As a site owner, you may have a preference on which image is used as an icon to represent the website. This tag could be used to specify the icon. For more, read this stack overflow discussion. +* __rel="next"__: If you have a paginated website and would like to provide hints to the search engine bots using this tag. An associated tag is rel="prev" for pointing back to the previous page. More details at Google Webmaster tools blog. + + +## Conclusion +In the Alexa Top 1000, <link> tags are heavily used for SEO purpose but, the majority of the usage is for geo-targeting. A lot of other nifty mechanisms are being put to use like specifying the pagination or the preferred icon image. However, these seem to be less popular than the basic use of targeting and canonicalization. + +Apart from SEO, the other use of the tag appears to be for performance optimization. Specially, "__[resource hints](https://w3c.github.io/resource-hints/)__" in the form of *dns-prefetch*, *preconnect* and *preload* are being used and they feature in the top 25 uses of <link rel> tag. To know more about this, have a look at [Yaov Weiss](https://twitter.com/yoavweiss?lang=en)' presentation from [Velocity New York](https://yoavweiss.github.io/velocity_nyc_resource_hints/#1). diff --git a/_posts/2016-12-13-Useful-SEO-Tags-Part-2.md b/_posts/2016-12-13-Useful-SEO-Tags-Part-2.md new file mode 100644 index 0000000000000..fbf9c8ac26d7a --- /dev/null +++ b/_posts/2016-12-13-Useful-SEO-Tags-Part-2.md @@ -0,0 +1,69 @@ +--- +layout: post +title: Useful SEO Tags - Part 2 +comment: true +--- + +In this second part of the blog series, I look at the distribution on the usage of the <meta> tags across the Alexa top 1000 website. + +## Introduction + +In the [first part of the blog](https://akshayranganath.github.io/Useful-SEO-tags-part-1/), I looked at the pattern of use for the  tag. In this blog post, I'd like to focus on the meta tags. If you'd like to see all the possible options, please refer the [meta-tags](http://www.metatags.org/all_metatags) website. + +## Meta Tags that matter +Moz has an excellent article by [Kate Morris](http://twitter.com/katemorris) on the set of [meta tags that matters](https://moz.com/blog/the-wonderful-world-of-seo-metatags). In this, it lists just 2 tags as essential. These are: + +* meta-description +* meta-content-type + +There are bunch of others that are listed as being optional and should be used only if the default behaviors have been changed. + +With this knowledge in hand, I wanted to check if the Alexa Top 1000 websites are sticking to the best practice or if there is a heavy use of the tags with no real returns. + +## Meta-Tag distribution +Let's start by looking at the top 25 meta-tags being used. I have published the entire distribution here: [https://gist.github.com/akshayranganath/ad0b170550714e2a77612bf0f81057da](https://gist.github.com/akshayranganath/ad0b170550714e2a77612bf0f81057da). + +| Attribute | Count | +| --------- | ------ | +| content | 14574 | +| name | 6205 | +| itemprop | 4316 | +| property | 3301 | +| http-equiv | 1219 | +| charset | 439 | +| itemscope | 214 | +| itemtype | 214 | +| itemid | 212 | +| data-reactid | 161 | +| id | 94 | +| class | 47 | +| data-app | 42 | +| value | 37 | +| data-type | 36 | +| lang | 23 | +| data-react-helmet | 17 | +| data-ephemeral | 17 | +| data-dynamic | 13 | +| xmlns:og | 12 | +| scheme | 11 | +| data-page-subject | 7 | +| xmlns:fb | 7 | +| data-ue-u | 7 | +| prefix | 3 | + +![meta tag distribution](/images/blog/seo_meta_tag_distribution.png) + +This is a simplified check since some use of the meta tag is by a combination of values. For example, the meta-description would look like this: + + + + +My first version of the script is does not capture this dependency but, I hope to add that capability over the next few weeks. At a very high level, it looks like most sites do follow the best practice. Schema.org tags are being heavily used and this makes sense due to the growing importance and the ability to control the behavior of results in SERPs. + +## Strange Use-Cases +I did see the meta tags being put into use for some strange uses. For example, and are not even required but, are quite heavily used. Here's a use that seems to make absolutely no sense + + + +## Conclusion +Meta tags appear to have been used by the top 1000 sites in the right intended manner, for most part. I plan to re-visit this and explore the usage oftag and break into the usage pattern. Stay tuned. \ No newline at end of file diff --git a/_posts/2016-12-6-Docker-Commands.md b/_posts/2016-12-6-Docker-Commands.md new file mode 100644 index 0000000000000..4e33c428d7821 --- /dev/null +++ b/_posts/2016-12-6-Docker-Commands.md @@ -0,0 +1,78 @@ +--- +layout: post +title: Docker Commands +comment: true +--- + +I completed the course on [Docker Quickstart](https://linuxacademy.com/devops/training/course/name/docker-quick-start) at LinuxAcademy today. Over the course of I made notes on the important commands. Here's my notes. + +## Installing docker +All commands as root on CentOs7 + + cd /etc/yum.repos.d + vim docker.repo + + [dockerrepo] + name=Docker Repository + baseurl=https://yum.dockerproject.com/repos/main/centos/7/ + enabled=true + gpgcheck=1 + gpgkey=https://yum.dockerproject.org/gpg + + yum install docker-engin + + systemctl enable docker + systemctl start docker + + usermod -a -G docker + cat /etc/group | grep docker #should use the as the group for the docker program + + +## Docker commands +List images + + docker images +List running docker processes + + docker ps +List all processes that were ever run + + docker ps -a +List only the container ids + + docker ps -a -q +Running processes + + docker run + docker run -d = run in disconnected / daemon mode + docker run --name="Some Name" = name the running instance + docker start = will restart a closed / exited instance of the image + docker exec -it = run a command within a running container without changing the state of the running container + docker stop = stop a running container by using the name + +Cleaning up docker + + docker rm containerid = removes an instance of the container that was run + + docker rm `docker ps -a -q` = remove all stopped containers + + docker rmi image-name = removes the docker image and its dependencies +Redirect port + + docker run -P = will redirect the container's port to a random port on the host machine's user port (port no 32,000+) + + docker run -p 8080:80 = will redirect the container's port 80 to a port 8080 on the host machine's user port + + docker port = will list the port mapping information +Adding volume +Using the "-v" option mounts the local file system. eg to mount for an nginx on centos + + -v /home/user/www:/usr/share/nginx/html + +Building a Docker file + + docker login --username= + Enter password + docker push username/repo + +Hope you find this useful as well! \ No newline at end of file diff --git a/_posts/2017-05-12-Akamai-Varnish-Connector-Training.md b/_posts/2017-05-12-Akamai-Varnish-Connector-Training.md new file mode 100644 index 0000000000000..357a7a21dd12c --- /dev/null +++ b/_posts/2017-05-12-Akamai-Varnish-Connector-Training.md @@ -0,0 +1,582 @@ +--- +layout: post +title: Notes from "Akamai Connector For Varnish" training +comment: true +description: Akamai and Varnish are launching a new solution called Akamai Connector For Varnish. Here's how to setup and work with Varnish. +--- + +Akamai and Varnish are launching a new solution called **[Akamai Connector For Varnish](https://developer.akamai.com/connector-for-varnish/)**. For this purpose, I took a training on Varnish and here are the notes from this session. + +## Varnish basic setup / commands + +![Varnish cache](https://www.smashingmagazine.com/wp-content/uploads/2013/12/varnish-cache-opt.png) + +## Default startup parameters +The startup parameters for Varnish are picked up from the file __/etc/varnish/varnish.params__. + +### Default VCL + +On a CentOS, the path is __/etc/varnish/default.vcl__. In this use case, I removed the "Via" and "Server" response header. + +```bash +sub vcl_deliver { + # Happens when we have all the pieces we need, and are about to send the + # response to the client. + # + # You can do accounting or modifying the final object here. + unset resp.http.server; + unset resp.http.via; + +} +``` + +### Check settings +To view the settings of current running Varnish instance, this command will help. + + varnishadm param.show + +To see the VCL that is currently loaded, use this command: + + varnishadm vcl.list + +In my case, the output was: + + active auto/warm 0 boot + +To see the parameter values for a given VCL, use this. In this case, I was using the default 'boot' VCL. + + varnishadm vcl.show -v boot + +## Varnish log +### Varnish log format + +Querying and grouping the logs + + varnishlog -q "RespHeader ~ 'Age: 0'" -g session + +Logs use a special language called _VSL-Query_. + +```bash +* << Session >> 44 +- Begin sess 0 HTTP/1 +- SessOpen ::1 36478 :80 ::1 80 1494529036.854837 21 +- Link req 45 rxreq +- SessClose REM_CLOSE 0.001 +- End +** << Request >> 45 +-- Begin req 44 rxreq +-- Timestamp Start: 1494529036.854865 0.000000 0.000000 +-- Timestamp Req: 1494529036.854865 0.000000 0.000000 +-- ReqStart ::1 36478 +-- ReqMethod GET +-- ReqURL / +-- ReqProtocol HTTP/1.1 +-- ReqHeader User-Agent: curl/7.29.0 +-- ReqHeader Host: localhost +-- ReqHeader Accept: */* +-- ReqHeader X-Forwarded-For: ::1 +-- VCL_call RECV +-- VCL_return hash +-- VCL_call HASH +-- VCL_return lookup +-- VCL_call MISS +-- VCL_return fetch +-- Link bereq 46 fetch +-- Timestamp Fetch: 1494529036.855280 0.000415 0.000415 +-- RespProtocol HTTP/1.1 +-- RespStatus 200 +-- RespReason OK +-- RespHeader Date: Thu, 11 May 2017 18:57:16 GMT +-- RespHeader Server: Apache/2.4.6 (CentOS) PHP/5.4.16 +-- RespHeader Last-Modified: Fri, 06 Jan 2017 10:17:18 GMT +-- RespHeader ETag: "fe-5456a52a3a62e" +-- RespHeader Content-Length: 254 +-- RespHeader Content-Type: text/html; charset=UTF-8 +-- RespHeader X-Varnish: 45 +-- RespHeader Age: 0 +-- RespHeader Via: 1.1 varnish-v4 +-- VCL_call DELIVER +-- RespUnset Server: Apache/2.4.6 (CentOS) PHP/5.4.16 +-- RespUnset Via: 1.1 varnish-v4 +-- VCL_return deliver +-- Timestamp Process: 1494529036.855288 0.000423 0.000008 +-- RespHeader Accept-Ranges: bytes +-- Debug "RES_MODE 2" +-- RespHeader Connection: keep-alive +-- Timestamp Resp: 1494529036.855311 0.000447 0.000023 +-- ReqAcct 73 0 73 258 254 512 +-- End +*** << BeReq >> 46 +--- Begin bereq 45 fetch +--- Timestamp Start: 1494529036.854903 0.000000 0.000000 +--- BereqMethod GET +--- BereqURL / +--- BereqProtocol HTTP/1.1 +--- BereqHeader User-Agent: curl/7.29.0 +--- BereqHeader Host: localhost +--- BereqHeader Accept: */* +--- BereqHeader X-Forwarded-For: ::1 +--- BereqHeader Accept-Encoding: gzip +--- BereqHeader X-Varnish: 46 +--- VCL_call BACKEND_FETCH +--- VCL_return fetch +--- BackendOpen 23 boot.default 127.0.0.1 8080 127.0.0.1 48074 +--- BackendStart 127.0.0.1 8080 +--- Timestamp Bereq: 1494529036.854966 0.000063 0.000063 +--- Timestamp Beresp: 1494529036.855216 0.000313 0.000249 +--- BerespProtocol HTTP/1.1 +--- BerespStatus 200 +--- BerespReason OK +--- BerespHeader Date: Thu, 11 May 2017 18:57:16 GMT +--- BerespHeader Server: Apache/2.4.6 (CentOS) PHP/5.4.16 +--- BerespHeader Last-Modified: Fri, 06 Jan 2017 10:17:18 GMT +--- BerespHeader ETag: "fe-5456a52a3a62e" +--- BerespHeader Accept-Ranges: bytes +--- BerespHeader Content-Length: 254 +--- BerespHeader Content-Type: text/html; charset=UTF-8 +--- TTL RFC 10 0 -1 1494529037 1494529037 1494529036 0 0 +--- VCL_call BACKEND_RESPONSE +--- VCL_return deliver +--- Storage malloc s0 +--- ObjProtocol HTTP/1.1 +--- ObjStatus 200 +--- ObjReason OK +--- ObjHeader Date: Thu, 11 May 2017 18:57:16 GMT +--- ObjHeader Server: Apache/2.4.6 (CentOS) PHP/5.4.16 +--- ObjHeader Last-Modified: Fri, 06 Jan 2017 10:17:18 GMT +--- ObjHeader ETag: "fe-5456a52a3a62e" +--- ObjHeader Content-Length: 254 +--- ObjHeader Content-Type: text/html; charset=UTF-8 +--- Fetch_Body 3 length stream +--- BackendReuse 23 boot.default +--- Timestamp BerespBody: 1494529036.855263 0.000360 0.000047 +--- Length 254 +--- BereqAcct 133 0 133 253 254 507 +--- End +``` + +### Vanish synthesized response + +```bash +curl -X PRI localhost -v +* About to connect() to localhost port 80 (#0) +* Trying ::1... +* Connected to localhost (::1) port 80 (#0) +> PRI / HTTP/1.1 +> User-Agent: curl/7.29.0 +> Host: localhost +> Accept: */* +> +< HTTP/1.1 405 Method Not Allowed +< Date: Thu, 11 May 2017 18:47:15 GMT +< Server: Varnish +< X-Varnish: 31 +< Content-Type: text/html; charset=utf-8 +< Retry-After: 5 +< Content-Length: 273 +< Connection: keep-alive +< + + + + 405 Method Not Allowed + + +

Error 405 Method Not Allowed

+

Method Not Allowed

+

Guru Meditation:

+

XID: 31

+
+

Varnish cache server

+ + +``` + +### Varnish piped response + +```bash +curl -X DUMMY localhost -v +* About to connect() to localhost port 80 (#0) +* Trying ::1... +* Connected to localhost (::1) port 80 (#0) +> DUMMY / HTTP/1.1 +> User-Agent: curl/7.29.0 +> Host: localhost +> Accept: */* +> +< HTTP/1.1 501 Not Implemented +< Date: Thu, 11 May 2017 18:47:58 GMT +< Server: Apache/2.4.6 (CentOS) PHP/5.4.16 +< Allow: GET,HEAD,POST,OPTIONS,TRACE +< Content-Length: 203 +< Connection: close +< Content-Type: text/html; charset=iso-8859-1 +< + + +501 Not Implemented + +

Not Implemented

+

DUMMY to /index.html not supported.
+

+ +* Closing connection 0 +``` + + +## Varnish Cache key +Cache key in varnish is based on "Host" and "URL". Varnish can __tag__ the cache by other parameters like "Vary", other headers or some other request base parameter. + +## Varnish status +__varnishstat__ can provide *ps* like interface. It provides a snapshot of current cache hit/miss and storage statistics. However, it is "snapshot" from the time the varnish has been started. + +However, there is no way to get historical data on cache rate. The only way is to request varnishstat every minute and then run a delta for generating the reports. + +## Purging mechanism + +4 main methods: + +- Ban +- Soft ban +- Purge +- Soft purge + +Hash key is formed by URL and path. Under this hash, there can be sub-hashes by custom header, condition, etc and objects remain under it. + +It is possible to remove all objects under one hash key or to surgically remove single sub-hash keyed object. + +It is possible to tag objects by version number. Eg: Add a custom header from origin called "X-Version" and use this in cached object. It is possible to purge all objects with this number. + + +## VCL Commands +Compile the VCL file to C program + + varnishd -C -f /etc/varnish/default.vcl + +Adding a new VCL + + varnishadm vcl.load boot2 /etc/varnish/default.vcl + +So 2 VCLs are available + + varnishadm vcl.list + active auto/warm 0 boot + available auto/warm 0 boot2 + +To use the new one, use this: + + varnishadm vcl.use boot2 + +To verify + + varnishadm vcl.list + available auto/warm 0 boot + active auto/warm 0 boot2 + + +## VCL + +- Goal of VCL is to have settings that are "un-crashable" +- Domain specific language +- State machine with functions as sttes +- built in functions start with __vcl_*__ +- **Intelligent purge**: It is possible to take a Purge request and after purge, the request can be reset. This way, the new object is fetched and possibly cached. +- "Retry-After" header can be easily added through VCL ==> useful for SEO. +- __Hit-for-Miss__ - a bit confusing. Basically, it is a state created to short-circuit the logic to detect uncacheable object. It is possible to set a TTL to store the evaluation saying an object should not be cached. +- **Normalizing queries**: it is possible to sort the query strings in the normalized order. This is the method: [STRING querysort(STRING)](https://varnish-cache.org/docs/4.0/reference/vmod_std.generated.html#func-querysort). + - For more granular way to fine-tune query strings, use [libvmod-querystring](https://github.com/Dridi/libvmod-querystring). + - Similar sorting can be done on cookies as well. + +Here's the VCL state diagram: +![VCL state machine](http://book.varnish-software.com/4.0/_images/simplified_fsm.svg) + + + +## ESI + +Akamai and Varnish support [Edge Side Includes](https://www.akamai.com/us/en/support/esi.jsp). Varnish's ESI implementation is a subset of features offered by Akamai. + +- ESI fragments are fetched sequentially in open source version +- In proprietary version, this fetch is done in parallel + +
+Only 3 tags are supported by Varnish:
+In Varnish we've only implemented a small subset of ESI. As of 2.1 we have three ESI statements:
+
+esi:include
+esi:remove
+<!--esi ...-->
+
+ +For more, see here: [Edge Side Includes](https://www.varnish-cache.org/docs/3.0/tutorial/esi.html). Here are the main tags necessary to trigger ESI. + +```bash +beresp.do_esi + Type: BOOL + Readable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error + Boolean. ESI-process the object after fetching it. Defaults to false. Set it to true to parse the object for ESI directives. Will only be honored if req.esi is true. + +req.esi + Type: BOOL + Readable from: client + Writable from: client + Boolean. Set to false to disable ESI processing regardless of any value in beresp.do_esi. Defaults to true. This variable is subject to change in future versions, you should avoid using + it. + + req.esi_level + Type: INT + Readable from: client + A count of how many levels of ESI requests we're currently at. +``` + +__Maximum includes = 5__ + +```bash +varnishadm param.show | grep esi +max_esi_depth 5 [levels] (default) +``` + +To make this more scalable, Varnish is moving towards [EdgeStash](https://info.varnish-software.com/blog/edgestash) + +## Edgestash + +![Varnish software](https://www.varnish-software.com/wp-content/uploads/320x200.png) + +Varnish supports server side rendering with EdgeStash. It is supported only on VarnishCache Plus. It is similar to Moustache/handlebar. + + +## VMOD + +>VMODs are extensions written for Varnish Cache. VMODs are complied into shared object (.so) and loaded into memory. [VMODs](https://www.varnish-cache.org/vmods/) + +Some interesting VMOD: +- *vmod_paywall* - can implement the functionality of paywall on websites +- *vmod_softpurge* - instead of default purge, try invalidation + + +## Cache invalidation + +Varnish offers different mechanisms of cache purge/invalidation. Here is a comparison of the implementation with Akamai's way of doing cache clearing/purge. + +| Purge | Soft Purge | Ban | Soft ban | Force Cache misses | Surrogate keys | +| -- | -- | -- | -- | -- | -- | +| Same as single object purge| Single object invalidation | ECCU purge | ECCU invalidation | Special | Cache tagging | + +Ban is preferred for multiple objects. It is async, if it does not use any __req__ parameter and uses only __obj__ parameters. + +**Force Cache miss **: You can fake a cache bypass. So the request does not check the cache for the cached object and hits the back-end. However, it has 2 behavior on response: + +- if backend fails, then, _this_ request fails and end user gets an error. Cached object is untouched. +- if backend succeeds, then the response replaces the existing cached object and the user gets this new response. + +Adding references to Akamai's APIs: +- *[Regular Purge / Invalidate](https://developer.akamai.com/api/purge/ccu/overview.html)*: This can be used for single URL or CP Code purge. +- *[ECCU Purge](https://control.akamai.com/dl/customers/other/CCU/Content_Control_Interfaces.pdf)*: This can be used for wildcarded URLs, and purging with complex conditions. + +Here's the complete purge and ban strategies allowed. I've taken it from the section [Purge - Bans - Cache Misses - Surrogate Keys](http://book.varnish-software.com/4.0/chapters/Cache_Invalidation.html). + + + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 17 Comparison Between: Purge, Softpurge, Bans, Force Cache Misses and Surrogate keys (hashtwo/xkey)
 PurgeSoft PurgeBansForce Cache MissesSurrogate keys
TargetsSpecific object (with all its variants)Specific object (with all its variants)Regex patternsOne specific object (with all its variants)All objects with a common hashtwo key
Frees memoryImmediatelyAfter grace timeAfter pattern is checked and matchedNoImmediately
ScalabilityHighHighHigh if used properlyHighHigh
CLINoNoYesNoNo
VCLYesYesYesYesYes
AvailabilityVarnish CacheVarnish CacheVarnish CacheVarnish CacheHashtwo VMOD in Varnish Plus 4.0 or xkey VMOD in Varnish Cache 4.1
+ + +### Common recommended VCLs + +- Drupal: [VCL purge & ban ](https://gitlab.wklive.net/snippets/32) +- Wordpress: [STEP-BY-STEP: SPEED UP WORDPRESS WITH VARNISH SOFTWARE](https://info.varnish-software.com/blog/step-step-speed-wordpress-varnish-software) +- Magento: [Using varnish with Magento](http://devdocs.magento.com/guides/v2.0/config-guide/varnish/config-varnish-magento.html) +- AEM: REPLACING ADOBE AEM [Part 1](https://info.varnish-software.com/blog/advanced-cache-invalidation-applied-replacing-adobe-aem-cq5-dispatcher-with-varnish-plus-part-1) and [Part 2](https://info.varnish-software.com/blog/advanced-cache-invalidation-part2) + +{Digression: Comparison of CMS platforms https://www.slideshare.net/TeroJuola/magento-2-hybris-websphere-commerce-oracle-atg-comparison } + +## Varnish Gather +A nifty tool, [Varnish Gather](https://github.com/varnish/varnishgather) to analyze and troubleshoot varnish information. + +If faced with an issue with Varnish setup, ask the customer to run this command and then get the full output. This can be used for detailed troubleshooting. + +eg; varnish lib should be mounted on tmpfs + + +## Varnish HA (VHA) +Basically it is an advanced implementation of [Internet Cache Protocol](https://tools.ietf.org/html/rfc2186). Here's details of VHA: https://www.varnish-software.com/plus/varnish-high-availability/. + +Once a single Varnish instance gets a new object to cache, it communicates to others that there is a new object. The other varnish instances then get the object from this first instance. + +### Varnish Probes +Varnish itself can send probes to back-end to mark it healthy / down. It can be configured to use a specific HTML page and the timers on connect time, poll time, health check % can be configured. + +Ways of splitting traffic to back-end. More info here: https://www.varnish-cache.org/docs/5.0/reference/vmod_directors.generated.html + +- fall back +- round robin +- shard + +>There are no loops in VCL. + +## CDNs using Varnish + +- Fastly +- MaxCDN +- Cachefly + +## Varnish Connector + +Working of Varnish connector is explained here. Basically, Varnish VCL can be setup such that it can: + +- drive the caching on Akamai +- purge issued to Varnish can clear Akamai cache as well + +![Varnish connector](https://developer.akamai.com/connector-for-varnish/images/architecture.png) + +### Installation + +First download the Akamai connector code from https://akamai-connector.varnish-software.com/dist/. + +To work with open source version, use this: + +```bash +yum install varnish-devel curl-devel openssl-devel python-docutils +``` + +To work with Varnishplus, use this + +```bash +yum install varnish-plus-devel curl-devel openssl-devel python-docutils +``` + +Then run the standard installation commands: +```bash +./configure +make +make install +``` + +To test, check if you have the Akamai connector help file: + + man vmod_akamai + +Now setup Akamai connector: + + akamai-connector-setup.sh install + +This will install a few VCL files +```bash +Akamai Connector for Varnish setup + +Installing + +Source: /usr/share/doc/akamai-connector/akamai_auto.vcl +Destination: /etc/varnish +Arguments: --no-clobber +Success + +Source: /usr/share/doc/akamai-connector/akamai.vcl +Destination: /etc/varnish +Arguments: --no-clobber +Success + +Source: /usr/share/doc/akamai-connector/akamai-connector.conf.example +Destination: /etc/akamai-connector.conf +Arguments: --no-clobber +Success +``` + +Add this to the default vcl - /etc/varnish/default.vcl +```perl +import std; +import akamai; +include "akamai_auto.vcl"; +``` + +After reloading, test if [Sureroute](https://developer.akamai.com/learn/Optimization/SureRoute.html) test object is returned. +```bash +curl -v localhost/akamai/testobject.html -H "Via:akamai.net(ghost) (AkamaiGHost)" -o /dev/null -s +* About to connect() to localhost port 80 (#0) +* Trying ::1... +* Connected to localhost (::1) port 80 (#0) +> GET /akamai/testobject.html HTTP/1.1 +> User-Agent: curl/7.29.0 +> Host: localhost +> Accept: */* +> Via:akamai.net(ghost) (AkamaiGHost) +> +< HTTP/1.1 200 OK +< Content-Length: 20480 +< Date: Fri, 12 May 2017 22:40:06 GMT +< X-Varnish: 2 +< Edge-Control: max-age=70 +< Cache-Control: post-check=60 +< Via: 1.1 varnish-v4, Akamai Connector/1.0.2 +< Accept-Ranges: bytes +< Connection: keep-alive +``` + +#### Gotchas +If Varnish connector is _NOT_ configured to purge Akamai, it will return with a 200 response code and an empty JSON _{}_. + +Varnish connector will tell downstream caches to cache for TTL. Akamai will cache for TTL+Grace. + + post-check + o Defines an interval in seconds after which an entity must be checked for freshness. The check may happen after the user is shown the resource but ensures that on the next roundtrip the cached copy will be up-to-date. \ No newline at end of file diff --git a/_posts/2017-06-10-quotes-from-devops-handbook.md b/_posts/2017-06-10-quotes-from-devops-handbook.md new file mode 100644 index 0000000000000..99bf9593510a3 --- /dev/null +++ b/_posts/2017-06-10-quotes-from-devops-handbook.md @@ -0,0 +1,398 @@ +--- +layout: post +title: Quotes from DevOps Handbook +comment: true +description: The DevOps Handbook is codification of DevOps mindset, practices and processes. Written as a follow-up to the other book, The Phoenix Project, it reinforces core concepts. Here are my favorite quotes from the book. +image: /images/DevOpsHandbook.png +tags: [devops, book, gene kim, jez humble, devops processes] +--- + +__[The DevOps Handbook]((https://www.amazon.com/DevOps-Handbook-World-Class-Reliability-Organizations/dp/1942788002)): How to create world-class agility, reliability and security in technology organizations__ is the follow-on book to __[The Phoenix Project](https://www.amazon.com/Phoenix-Project-DevOps-Helping-Business/dp/0988262509/ref=pd_lpo_sbs_14_img_0?_encoding=UTF8&psc=1&refRID=6Q7CNB17CF6AAX0DWSKT): A Novel about IT, DevOps, and Helping Your Business Win__. I had collected my favorite [quotes from the Phoenix project](https://akshayranganath.github.io/quotes-from-book-the-phoenix-project/) earlier. I finally managed to get the DevOps handbook and here's my collection of favorite thoughts from the book. + +![DevOps Handbook cover photo](/images/blog/DevOpsHandbook.png) + +## DevOps Myths + +- DevOps is only for startups +- DevOps replaces Agile +- DevOps is incompatible with ITIL +- DevOps is incompatible with Information Security and Compliance +- DevOps means eliminating IT Operations, or _"NoOps"_ +- DevOps is just _"Infrastructure as Code"_ +- DevOps is only for open source software + +### DevOps Aha! + +>..DevOps is a manifestation of creating dynamic, learning organizations that continually reinforce high-trust cultural norms. + +DevOps is the outcome of applying the most trusted principles from the domains of physical manufacturing and leadership to the IT value stream. + +### Technical Debt +..describes how decisions we make lead to problems that get increasingly more difficult to fix over time, continually reducing our available options in the future. + +### Problem with traditional Dev & Ops Structures +Talking about the discussion in the book, [The Goal](https://www.amazon.com/Goal-Process-Ongoing-Improvement/dp/0884270610) about conflict between traditional organizational goals and metrics, + +>the core, chronic effect - when organization measurements and incentives across different silos prevent the achievement of global organizational goals. + +#### Downward spiral +.. the systems most prone to failure are our most important and at the epicenter of our most urgent changes. + +_Why is this important?_ +>..every IT organization has two opposing goals, and second, every company is a technology company, whether they know or not. + +_There is a discussion of *dark releases* and *feature flags*. Here's a very well written description by [Martin Fowler](https://twitter.com/martinfowler):_ [Feature toggles](https://martinfowler.com/articles/feature-toggles.html) + +![feature toggles overview](/images/blog/feature_toggles.png) + +### Where is DevOps impact seen? +This list is based on work done for [Puppet Labs - State of DevOps report](https://puppet.com/resources/whitepaper/2016-state-of-devops-report). + +- Throughput metrics +- Code change and code deployments (thirty times more frequent) +- Code and change deployment lead time (two hundred times faster) +- Reliability metrics +- Production deployments (sixty times higher change success rate) +- Mean time to restore services (168 times faster) +- Organizational performance metrics +- Productivity, market share and profitability goals (two times more likely to exceed) +- Market capitalization growth (50% higher over three years) + +## Technology Value Stream + +>.. is defined as the process required to convert a business hypothesis into a technology enabled service that delivers value to the customer. + +``` +My paraphrasing: +Value stream starts when the any engineer checks +in a change (could be design, user story) and ends with this change +is successfully running in production and providing value +to customer and generating feedback and telemetry. +``` + +### Why is the focus on automation? + +*Design and development* is akin to __Lean Product Development__ and is highly variable and highly uncertain, often requiring high degrees of creativity and work that may never be performed again, resulting in high variability of process time. In contrast, the second phase of work which includes *Testing and Operations* is akin to __Lean Manufacturing__. It requires creativity and expertise, and strives to be predictable and mechanistic, with the goal of achieving work outputs with minimized variability. + +``` +End goal: +Have testing and operations simultaneously with design and development to enable fast flow of work. +``` + +#### Concept of Lead Time + +``` + |<--------------- Lead Time ---------------->| + |=======================|====================| +Ticket Work Work +Created Started Completed + |<-- Process Time -->| +``` + + +>When we have long deployment lead times, heroics are required at almost every stage of the value stream. + + +## The Three Ways +Taken from the blog by [Gene Kim](https://twitter.com/RealGeneKim) titled [The Three Ways: The Principles Underpinning DevOps](https://itrevolution.com/the-three-ways-principles-underpinning-devops/) + +### First Way - Fast Flow + +- *First Way*: The First Way requires the fast and smooth flow of work from Development to Operations, to deliver value to customers quickly. We optimize for this global goal instead of local goals, such as Development feature completion rates, test find/fix rations, or Ops availably measures. + +![first way](/images/blog/first_way.png) + +>Stop starting. Start finishing. - David J. Anderson. + +__Issue with hand-offs__ +With enough handoffs, the work can completely loose the context of the problem being solved or the organizational goal being supported. + +__5 Focusing steps - Dr Goldratt__ + +![The Goal book cover](/images/the_goal_book.jpg) + +- identify the system's constraint +- decide how to explot the system's contraint +- subordinate everything else to the above decisions +- elevate the sytem's constraint +- if in previous steps a constraint has been broken, go back to step one, but do not allow inertia to cause a system constraint. + +__DevOps Transformation - Focus Areas__ + +- Environment creation +- Code deployment +- Test setup and run +- Overly tight architecture + +__I am paraphrasing here__ +The goal of fast flow is to make any kind of waste and hardships - anything that requires heroics is made visible and to systematically alleviate or eliminate the burden. Types of waste that lead to heroics are: + +- partially done work (sitting in QA, waiting for deployment - stuck WIP) +- extra processes (eg Documenting steps to document - I guess this is where CMM models failed) +- extra features (adding more things than a minimum viable product {MVP}) +- task switching +- waiting +- motion (lack of co-located colleagues, hand-offs, KTs) +- defects +- nonstandard or manual work + +### Second way - Principles of Feedback + +![second way](/images/blog/second_way.png) + +.. Second way describes the principles that enable the fast and constant feedback from right to left at all stages of value stream. + +According to Dr Spear, the goal of *swarming* is to contain problems before they have a chance to spread, and to diagnose and treat the problem so that it cannot recur. + +``` +When automated build is used and a test fails, the entire pipeline fails - and that is acceptable. Here's the reason. +``` +Preventing the introduction of new work enables continuous integration and deployment, which is a single piece of flow in the technology value stream. All changes that pass our continuous build and integration tests are automatically deployed into production, and any changes that cause any tests to fail trigger our Andon cord and are swarmed until resolved. + +__Biggest differentiator in devOps compared to traditional software life-cycle?__ +In the technology value stream, we optimize for downstream work centers by designing for operations, where operational non-functional requirements (e.g., architecture, performance, reliability, stability, testability, configurability, and security) are prioritized as highly as user features. + + +Preventing the introduction of new work enables continuous integration and deployment, which is + +### Third way - Continual learning and Experimentation + +![third way](/images/blog/third_way.png) + +.. the Third Way focuses on creating a culture of continual learning and experimentation. These are the principles that enable constant creation of individual knowledge, which is then turned into team and organizational knowledge. + +__How?__ + +We improve daily work by explicitly reserving time to pay down technical debt, fix defects, and refactor and improve problematic areas of our code and environments. + +### Selecting the correct Value Stream + +Greenfield / brownfield do not matter - as long as the application is architected (or re-architected) for testability and deployability. + +Although _systems of engagement_ are more easy to adopt to devOps, it is the _systems of record_ that can form a bottleneck. So everything in the org has to move to devOps going against the _bimodal_ IT design. + + +### Understanding work +Next step in value stream is: + + ..what work is performed and by whom, and what steps can we take to improve flow + +Typical teams involved in value stream mapping: + +- Product owners +- Development +- QA +- Operations +- InfoSec +- Release Managers +- Technology executive or value stream manager + +___Managing Technical Debt and Non-functional requirements___ +Invest around 20% of all dev and ops time on Non-functional requirements. This is the only way to pay down technical debt. + +#### Use tools to reinforce behavior + +Tools should be designed such that dev and Ops use the same systems to create a queue of shared work. It can show a unified backlog. This can help in prioritizing work that delivers highest value to the organization. + +## Designing Organizational Structure + +_Dr. Melvin Conway - Conway's Law:_ +>“organizations which design systems...are constrained to produce designs which are copies of the communication structures of these organizations….The larger an organization is, the less flexibility it has and the more pronounced the phenomenon.” + +Simplified by Eris S. Raymong: +>“The organization of the software and the organization of the software team will be congruent; commonly stated as ‘if you have four groups working on a compiler, you’ll get a 4-pass compiler.’” + +### Organizational Archetypes + +- **Functional-oriented organizations** optimize for expertise, division of labor, or reducing cost. These organizations centralize expertise, which helps enable career growth and skill development, and often have tall hierarchical organizational structures. +- **Market-oriented organizations** optimize for responding quickly to customer needs. These organizations tend to be flat, composed of multiple, cross-functional disciplines .. which often lead to potential redundancies across the organization. (__most commonly seen in early adopters of devOps). +- **Matrix-oriented organizations** attempt to combine functional and market orientation. However, as many who work in or manage matrix organizations observe, matrix organizations often result in complicated organizational structures + +#### Why Functional oriented organizations fail to adopt devOps? +.. as we increase the number of Development teams and their deployment and release frequencies, most functionally-oriented organizations will have difficulty keeping up and delivering satisfactory outcomes, especially when their work is being performed manually. + +**How to adopt Market organization practices?** +Instead of re-organizing, embed functional specialist into service teams or automate the specific process (like provisioning, testing). This will help service teams to deliver value independent of other groups. + +**Common Trend in good DevOps organizations** +> ..a high-trust culture that enables all departments to work together effectively, where all work is transparently prioritized and there is sufficient slack in the system to allow high-priority work to be completed quickly. + +**Issue of monolithic application** +>When we have a tightly-coupled architecture, small changes can result in large scale failures. As a result, anyone working in one part of the system must constantly coordinate with anyone else working in another part of the system they may affect, including navigating complex and bureaucratic change management processes. + +**Bounded Contexts** +.. described in __Domain Driven Design__ by __Eric J. Evans__ +>The idea is that developers should be able to understand and update the code of a service without knowing anything about the internals of its peer services. Services interact with their peers strictly through APIs and thus don’t share data structures, database schemata, or other internal representations of objects. + +**Heather Mickman - Requirements from team members** +>Because our team also needed to deliver capabilities in days, not months, I needed a team who could do the work, not give it to contractors—we wanted people with kickass engineering skills, not people who knew how to manage contracts. + +## Building the foundation + +One of the recurring idea in building the foundation is for Operations team to build self-service capabilities for the Dev teams. Basically, Dev teams should rely on Ops tools rather than on Ops people. The tools should be completely automated and the usage should be on "on demand" basis. Dev teams should not have to use ticketing system or it'll lead to bottle necks. + + +**Breaking the Dev / Ops boundaries** +It is better to either: + +- embed Ops engineers into Dev teams OR +- assign Ops liason to Dev teams + +This will ensure that Ops priorities are now driven by the goals of the product teams as opposed to Ops focused inwardly on solving their own problems. It has a side-effect where the interaction can help the dev teams understand the process and open up automation opportunities. + +### Scrum Standup meetings + +Popular ritual of Scrum is the Daily Standup meeting. Discuss: + +- what was done yesterday +- what is going to be done today +- what is preventing you from getting your work done + +### Retrospectives + +![rear view](/images/retrospect.jpg) + +An agile practice where teams meet at the end of a development interval and discuss 3 things: + +- what was successful +- what can be improved +- how to incorporate the success and improvements in the future iterations + +**Building Kanbans for Operations** + +Making the work of ops visible ensures that it is easier to keep track of the work in progress, the effort required to move the code to production and the points where back-logs occur and potentially improve the process. + +## The First Way - Practices of Flow +_my summary_ +To help in a fast deployments, the basic building blocks are: + +- create the foundation for an automated deployment pipeline +- automate testing +- trunk based development +- enable low-risk releases and very find-grained telemetry + +>In Puppet Labs' 2014 _State of DevOps Report the use of version control by Ops was the highest predictor of both IT performance and organizational performance. In fact, whether Ops used version control was a higher predictor for both IT performance and organizational performance than whether Dev used version control. + +Why is this so? It's because there are a lot more complex details involved in configurable settings than in the application code. + +__Easier to build and throw-away infrastructure__ +>Bill Baker, a distinguished engineer at Microsoft, quipped that we used to treat servers like pets: “You name them and when they get sick, you nurse them back to health. [Now] servers are [treated] like cattle. You number them and when they get sick, you shoot them.” + +__What is _done_?__ +"Done" should be re-defined to "running in production-like" environment. +Get away from a feature being called done when it runs on a developer machine. + +### Fast & Reliable Automated testing + +Warning bells to IT outsourcing companies specializing in just testing services? +>we are likely to get undesired outcomes if we find and fix errors in a separate test phase, executed by a separate QA department only after all development has been completed. + +__Automated testing process __ _my summary_ +Automated testing ensured quality is built at the earliest stage into the product. By having automated unit tests, errors are found at an earlier stage and avoids re-work. Even when a bug is found at a later stage, the remedial process should be to constructed an equivalent unit test to catch a similar bug. Logging of the tests and its results automatically collects evidence for audit and compliance requirements. Integration tests tend to be brittle and it is always better to find as many bugs during unit and acceptance testing. + +Refer to the excellent blog post by [Martin Fowler](https://twitter.com/martinfowler?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Eauthor) titled [Test Pyramid](https://martinfowler.com/bliki/TestPyramid.html). + +![test pyramid](/images/AgileTestingPyramid2.jpg "Testing pyramid by Martin Fowler") +__Goal of Deployment Pipeline__ +>The goal of the deployment pipeline is to provide everyone in the value stream, especially developers, the fastest possible feedback that a change has taken us out of a deployable state. + +>When we use infrastructure as code configuration management tools, we can use the same testing framework that we use to test our code to also test that our environments are configured and operating correctly. + +### Enable and Practice Continuous Integration + +_Trunk based development_ (__my summary__) +The more time developers work on a branch, the more difficult it is to merge back. Since it is hard to merge, we tend to delay and this leads to self-defeating cycle. + +>Without automated testing, continuous integration is the fastest way to get a big pile of junk that never compiles or runs correctly.
- [Gary Gruver](https://twitter.com/GRUVERGary) + +** Version control & Continuous Integration ** +..after comprehensive use of version control, continuous integration is one of the most critical practices that enable the fast flow of work in our value stream, enabling many development teams to independently develop, test, and deliver value. + + +__Gated Commits__: Basically, verifying the code _before_ it is merged into trunk. When a developer commits code, it undergoes the automated testing and only if it passes, code is merged into trunk. If it fails, the developer is notified and the merge fails. This ensures that the trunk is always tested code. + +>trunk-based development predicts higher throughput and better stability, and even higher job satisfaction and lower rates of burnout
- State of DevOps, 2015 + +### Automate and enable low-risk releases + +_Automated deployments_: Document the steps in deployment process and automate as much as possible. + +__3 characteristics of automated deployments__ + +- _Deploying the same way to every environment_: By using the same steps for deployment in each environment, the process is tested and stable. +- _Smoke testing our deployments_: All the components should and critical transactions should be tested to ensure all components are tested. +- _Ensure we maintain consistent environment_: Ensure that the environments are synchronized. + +__Decouple release from deployment__ + +- _Deployment_ is the installation of a specified version of software to a given environment .. a deployment may or may not be associated with a release of a feature to customers. +- _Release_ is when we make a feature (or set of features) available to all our customers or a segment of customers + + +## Release Patterns +2 broad ways of release pattern: + +- Environment-based release patterns + - Blue / Green + - Canary deployments: Typical deployment pattern: + - A1 group: In prod, but serve to internal users only + - A2 group: small % of customers + - A3 group: Rest of production servers + +- Application-based release patterns + - Feature flag + - Dark launches + +__Cluster immune system__ extends on canary release by linking production monitoring to releae process. It'll automativally roll back change when user-performance issues are detected. + +__Feature toggle__ enables us to do the following: + +- Roll back easily +- Gracefully degrade performance +- Increase our resilience through a service-oriented architecture + + +__Testing tip__ +>To ensure that we find errors in features wrapped in feature toggles, our automated acceptance tests should run with all feature toggles on. + +Ultimate goal of CI/CD effort: +.. deployments should be low-risk push-button events we can perform on demand. + +Paradox of goals in IT Organizations: +>reducing our overall complexity and increasing the productivity of all our development teams is rarely the goal of an individual project. + +#### Replacing monolith +2 patterns: + +- [Strangler application](https://www.martinfowler.com/bliki/StranglerApplication.html) pattern +- _[Branching by abstraction](https://martinfowler.com/bliki/BranchByAbstraction.html)_ pattern by Paul Hammant + +## The Second Way + +__Secret of high performers__ +>High performers used a disciplined approach to solving problems, using production telemetry to understand possible contributing factors to focus their problem solving, as opposed to lower performers who would blindly reboot servers. + +__Telemetry__ defined +>an automated communications process by which measurements and other data are collected at remote points and are subsequently transmitted to receiving equipment for monitoring +.. telemetry is what enables us to assemble our best understanding of reality and detect when our understanding of reality is incorrect. + +In the world of devOps, monitoring is so important that monitoring systems need to be more available and scalable than the systems being monitored. + +__Culture of blame__ +Typical metric used is *[mean time until declared innocent](http://www.networkworld.com/article/2264981/lan-wan/eliminating-the-mean-time-to-innocence.html)* - how quickl can we convince everyone else that we didn't cause the outage. + +#### Information radiator +"[Information radiator](https://www.agilealliance.org/glossary/information-radiators/)" is the generic term for any of a number of handwritten, drawn, printed or electronic displays which a team places in a highly visible location, so that all team members as well as passers-by can see the latest information at a glance: count of automated tests, velocity, incident reports, continuous integration status, and so on. + +The goal of telemetry and monitoring is to reduce MTTR instead of trying to focus and reduce MTBF. + +![monitoring break down](https://image.slidesharecdn.com/monitoring-project-overview-120119004841-phpapp02/95/atlassian-hostedops-ondemand-monitoring-6-728.jpg?cb=1327832130) + +> Alert fatigue is the single biggest problem we have right now.. We need to be more intelligent about our alerts or we'll go insane. + +## Tools +- statsD +- Zookeeper +- Etcd +- Consul +- stats - mean, median, std deviaton, chi squared distribution, fast fourier transform diff --git a/_posts/2017-09-15-OCSP-Validation-With-Openssl.md b/_posts/2017-09-15-OCSP-Validation-With-Openssl.md new file mode 100644 index 0000000000000..a1a43b5a4fd66 --- /dev/null +++ b/_posts/2017-09-15-OCSP-Validation-With-Openssl.md @@ -0,0 +1,192 @@ +--- +layout: post +title: OCSP Validation with OpenSSL +comment: true +description: OCSP Stapling is becoming pervelant across browsers for validating certificates. Here I show how to run this validation manually with OpenSSL. +image: /images/blog/tls-validation.png +tags: [ocsp, tls, openssl, security] +--- + +>Edit June 19,2020: Added details to support OCSP request to servers that don't support HTTP 1.0. + +I had been working on understanding and troubleshooting an OCSP implementation and learnt a few things and thought I could share them on the blog. + +## What is OCSP? + +Online Certificate Status Protocol (OCSP) defined in [RFC 2560](https://www.ietf.org/rfc/rfc2560.txt) is a protocol that: +>enables applications to determine the (revocation) state of an identified certificate. OCSP may be used to satisfy some of the operational requirements of providing more timely revocation information than is possible with CRLs and may also be used to obtain additional status information. An OCSP client issues a status request to an OCSP responder and suspends acceptance of the certificate in question until the responder provides a response. + +Basically, OCSP is a mechanism where a client can ask the CA if a certificate is valid. This method is better than Certificate Revocation List (CRL). In the CRL method, the CA publishes a list of all the certificates that it has issues and that has now been revoked. Instead of processing this whole bunch, the client can check the status of just one certificate with OCSP. + +Here's the steps of OCSP, as explained in the [OCSP Stapling blog by Mozilla](https://blog.mozilla.org/security/2013/07/29/ocsp-stapling-in-firefox/). + +![OCSP request](https://blog.mozilla.org/security/files/2013/08/p4.png) + +## What is OCSP Stapling? + +With OCSP, the client is responsible to make a call to the CA (OCSP responder) to verify the status of a server certificate. This can cause additional round-trip delays. So an alternate solution was designed where the server could help. When the client initiates the TLS hand-shake, the server can include the OCSP validation message along with its certificate. By "stapling" the verification information, the client can complete the hand-shake faster. + +OCSP stapling was originally defined as Transport Layer Extension in [RFC 6066](https://www.ietf.org/rfc/rfc6066.txt). Specifically, the RFC calls out this functionality: +>Allow TLS clients and servers to negotiate that the server sends the client certificate status information (e.g., an Online Certificate Status Protocol (OCSP) [RFC2560] response) during a TLS handshake. This functionality is desirable in order to avoid sending a Certificate Revocation List (CRL) over a constrained access network and therefore saving bandwidth. + +Again, here's the working of OCSP stapling from [OCSP Stapling blog by Mozilla](https://blog.mozilla.org/security/2013/07/29/ocsp-stapling-in-firefox/). + +![OCSP stapling](https://blog.mozilla.org/security/files/2013/08/p6.png) +OCSP verification has been around for some time and OCSP stapling has been baked into the browsers for the past few years now. It is now gaining more widespread use as IoT devices are started to implement mutual authentication schemes. + + +## Testing OCSP with Openssl +I had been working on an implementation that uses this OCSP Stapled response. The use case was that connected device makes a request to server over TLS. The device presents a client cert to authenticate itself to the server. The server verifies and then responds back with its certificate and the stapled OCSP response for the client to authenticate. + +We ran into issues over the stapling and we had to verify the result. For this purpose, I am showing a request/response that does not include client certificates. This just makes the discussion a little bit simple. + +To work on this aspect, I started to use Openssl and here's the steps to achieve it: + +### Step 1: Get the server certificate +First, make a request to get the server certificate. When using ```openssl s_client -connect``` command, this is the stuff between the ```------BEGIN CERTIFICATE-----``` and ```-----END CERTIFICATE-----```. I am using [www.akamai.com](www.akamai.com) as the server. + +```bash +openssl s_client -connect www.akamai.com:443 < /dev/null 2>&1 | sed -n '/-----BEGIN/,/-----END/p' > certificate.pem +``` + +The server certificate is saved as _certificate.pem_. + +### Step 2: Get the intermediate certificate +Normally, a CA does not sign a certificate directly. They use intermediaries and we need to this make the openssl command work. So, make a request to get all the intermediaries. + +To view the list of intermediate certs, use the following command. + +```bash +openssl s_client -showcerts -connect www.akamai.com:443 < /dev/null 2>&1 | sed -n '/-----BEGIN/,/-----END/p' +``` + +The very first certificate is the server certificate we saved in step 2. For all the certificates below it, copy and save to a file named _chain.pem_. + +### Step 3: Get the OCSP responder for server certificate +The next step is to get the OCSP responder information. There are two ways to do this: + +#### OCSP Responder with a command +We can use the server certificate _certificate.pem_ and run a command to extract just the OCSP responder field: + +```bash +$ openssl x509 -noout -ocsp_uri -in certificate.pem +http://ss.symcd.com +``` + +So here, [http://ss.symcd.com](http://ss.symcd.com) is the OCSP responder. + +#### OCSP Responder by examination +We can use the openssl command to print all the server certificate information using this command: + +```bash +openssl x509 -text -noout -in certificate.pem +``` + +In the response, look for the section named _Authority Information Access_. This will hold the OCSP responder URL. In this case, here's what I see: + +```bash + Authority Information Access: + OCSP - URI:http://ss.symcd.com + CA Issuers - URI:http://ss.symcb.com/ss.crt +``` + +### Step 4: Make the OCSP request + +Now that we have the server certificate, CA certificate chain and the OCSP responder URL, we can make the actual verification call. + +```bash +openssl ocsp -issuer chain.pem -cert certificate.pem -text -url http://ss.symcd.com +``` + +Here's the relevant part of the response. +```bash +OCSP Request Data: + Version: 1 (0x0) + Requestor List: + Certificate ID: + Hash Algorithm: sha1 + Issuer Name Hash: D1B1648B8C9F0DD16BA38ACD2B5017D5F9CFC064 + Issuer Key Hash: 5F60CF619055DF8443148A602AB2F57AF44318EF + Serial Number: 7E34F9C3D2D21D999B668CC6C80EA4A8 + Request Extensions: + OCSP Nonce: + 0410AE9A77D9FBDE8F7EC1F0C3305183BAEC +OCSP Response Data: + OCSP Response Status: successful (0x0) + Response Type: Basic OCSP Response + Version: 1 (0x0) + Responder Id: A2117BB82AC97305A9CCC170F80F82442BBF509A + Produced At: Sep 12 16:58:10 2017 GMT + Responses: + Certificate ID: + Hash Algorithm: sha1 + Issuer Name Hash: D1B1648B8C9F0DD16BA38ACD2B5017D5F9CFC064 + Issuer Key Hash: 5F60CF619055DF8443148A602AB2F57AF44318EF + Serial Number: 7E34F9C3D2D21D999B668CC6C80EA4A8 + Cert Status: good + This Update: Sep 12 16:58:10 2017 GMT + Next Update: Sep 19 16:58:10 2017 GMT + +``` + +The most important part is the __Cert Status: good__ line. This indicates that everything is kosher and the client can trust the certificate. The other part of interest is the details __Next Update__. This indicates the OCSP stapling response can be cached until the time so that we don't overload the OCSP responder. + +### Handling newer OCSP validators + +Many new OCPS validation endpoints like `ocsp2.globalsign.com` don't support HTTP 1.0 requests. OCSP by default uses HTTP 1.0 which does not mandate the `Host` header. When this happens you may see a `400` error like this: + +``` +OCSP Request Data: + Version: 1 (0x0) + Requestor List: + Certificate ID: + Hash Algorithm: sha1 + Issuer Name Hash: 12EADF46CC0880387360B65A691601CC0CB5E9E2 + Issuer Key Hash: A92B87E1CE24473B1BBFCF853702559D0D9458E6 + Serial Number: 5625521AFA513B6D970FFAC1 + Request Extensions: + OCSP Nonce: + 0410C43AA490D8782F80C2D0751C417B6486 +Error querying OCSP responder +4732960364:error:27FFF072:OCSP routines:CRYPTO_internal:server response error:/AppleInternal/BuildRoot/Library/Caches/com.apple.xbs/Sources/libressl/libressl-47.120.1/libressl-2.8/crypto/ocsp/ocsp_ht.c:251:Code=400,Reason=Bad Request +``` + +To fix this mismatch, we will need to add the Host header. This can be achieved by adding the option `-header` followed by the header information. + +Here's an example command: + +``` +openssl ocsp -issuer chain.pem -cert certificate.pem -text -url http://ocsp2.globalsign.com/cloudsslsha2g3 -header "HOST" "ocsp2.globalsign.com" +``` + +Thanks to [Jim Carter's explanation](http://www.jfcarter.net/~jimc/documents/bugfix/21-openssl-ocsp.html) for helping identify and fix the issue. + +## Bonus: Dissecting OCSP request +I was curious to see on what actually happens during the OCSP request and ran a wireshark trace. When we make the OCSP request, this is submitted as a HTTP POST. In this case, the headers looked like this: + +![wireshark trace](/images/blog/ocsp_wireshark.png) + +```bash +POST / HTTP/1.0 +Content-Type: application/ocsp-request +Content-Length: 120 + +0v0t0M0K0I0 ..+..........d... +.k...+P.....d.._`.a.U..C..`*..z.C....~4.......f.......#0!0.. +.....0........w....~...0Q... + +HTTP/1.0 200 OK +Server: nginx/1.10.2 +Content-Type: application/ocsp-response +Content-Length: 1609 +content-transfer-encoding: binary +Cache-Control: max-age=329052, public, no-transform, must-revalidate +Last-Modified: Tue, 12 Sep 2017 16:58:10 GMT +Expires: Tue, 19 Sep 2017 16:58:10 GMT +Date: Fri, 15 Sep 2017 21:33:58 GMT +Connection: close +``` +If you notice, the "Cache-Control: max-age=329052" will be equal to the same as telling the client to cache the response until the __Next Update__ time period. So the POST request can be cached. + +## Conclusion +OCSP request/response sounds a bit complex but, ultimately it is just HTTP request / response and can be tested with simple command line tools. + diff --git a/_posts/2017-1-18-what-metrics-matter-for-seo.md b/_posts/2017-1-18-what-metrics-matter-for-seo.md new file mode 100644 index 0000000000000..9433caafef013 --- /dev/null +++ b/_posts/2017-1-18-what-metrics-matter-for-seo.md @@ -0,0 +1,95 @@ +--- +layout: post +title: What metrics matter for SEO? +comment: true +--- + +*Google is very nuanced in the way it handles the Site speed. It appears to rely on some combination of TTFB coupled with rendering metric like Time to first paint / start render or DomInteractive. However, it is very hard to find the exact metric. So focus on delivering the best performance to user and Google will automatically rank you well!* + +![measurement](/images/blog/measurement.jpg) + +## Background +Before giving away the answer on which metric matters, let's first study the published studies around this topic. + +Recently, there was a study published that seemed with the title [Does Site Speed Really Matter to SEO?](https://www.goivvy.com/blog/site-speed-matter-seo). Its main conclusion was that TTFB is the metric that strongly correlates to a higher google ranking. So, this was generally considered to be the smoking gun and the metric that needs to be optimized if you want a better ranking on Google. + +A study pretty similar to this one was published a few years back by Zoompf. The study was summarized on the Moz site under the title [How Website Speed Actually Impacts Search Ranking](https://moz.com/blog/how-website-speed-actually-impacts-search-ranking). Again, the basic premise was that TTFB had a much higher correlation with Google ranking. Specifically, a site with a lower TTFB would be ranked higher on Google SERP (Search Engine Results Page). Metrics like Time to DocComplete and RenderTime were considered to have low or no correlation to the actual ranking by Google. + +The third and more interesting study was published by an SEO analyst with a study titled [Does Speed Impact Rankings?](http://neilpatel.com/blog/does-speed-impact-rankings/). In this study the author concludes that: + +- Although speed is important, it is just one of the many ranking factors +- User interface plays a big role. Create a very minimalistic interface may not help in better google ranking +- If you're starting on optimizing, start with TTFB + +I like this study since the reason TTFB is considered important is that it is easier to measure and independent of the browser. All other metrics like Start Render, DomInteractive, etc rely on specific browser implementations. So optimizing the metric for say Chrome may or may not impact the actual SERP. However, optimizing TTFB would impact each and every user and bot. + +Now, let's dive a bit more into the nitty-gritties of what matters for SEO. + +## Sampling Bias? +Although I highly value the studies done by each of the above authors, there are some unexplained issues or problems with methodology. The first study clears hints to this under the section Tail wagging the doc. Let me quote the statement: +
Do these websites rank highly because they have better back-end infrastructure than other sites? Or do they need better back-end infrastructure to handle the load of ALREADY being ranked higher?
+In other words, all of the studies have an issue where it's hard to go ignore this: + +![Correlation is not causation](/images/blog/correlation_not_causation.jpg) + +(Digressing: If you want to read up on more serious issues related to correlation is not causation issue, head [over here](http://www.skepticalraptor.com/skepticalraptorblog.php/correlation-does-not-imply-causation-except-when-it-does/)). + +To summarize, here's the issues with the studies: + +- It only looks at one factor like TTFB: Pages from big companies could have higher TTFB but, it could also have very complex page that is favored by Google over similar speed but lesser complex page. +- It ignores the actual end user behavior: Suppose, there are 2 websites discussing the strategy used by 2 teams in a football match. Site A has detailed paragraphs while Site B has short summary followed by graphs, pictures and images. Site A could start out being ranked higher due to TTFB/start render but over time, Google will learn from user behavior and rank Site B higher +- It fails to combine the metrics for holistic view: TTFB may impact sitespeed. However, what if TTFB combined with start render and number of images on page is the real reason for higher ranking? The last factor is not even part of the analysis. +- Sampling bias in research terms: If the researchers had used terms like "tesla", there is a very high chance that brands associated with the name will rank higher, regardless of site performance. This is simply due to relevance. It is unclear on how well the terms were selected and if they were devoid of any such terms. Even if the search terms are filtered, Google has started to answer questions directly in search result page that ranking may have no impact. eg: If you search sampling bias, there is snippet explaining it and I never have to click on the results at all. + +Due to these complexities, I wanted to explore a more robust and continual study and found the research by Moz pretty interesting. + +## Study by Moz +Moz publishes a study called Search Engine Ranking Factors. They use a combination of correlation data and survey data to get the results. This is quite a unique way of presenting the information. + +First, let's look at the correlation results. + +## Moz: Correlation data +Since I am at Akamai, my focus is on the metric associated with the Page-Level Keyword Agnostic Features. and Domain level Keyword-agnostic Features. Of this, a few things stand out: + +- Server response time for this URL +- URL is HTTPS +- URL length +- URL has hyphens +- Domain has numbers +- Length of domain name and length including sub-domains + +Apart from this, bulk of the metrics are related to the actual page content, links to the page, social interactions and mentions. + +Now, let's look at the survey data. +## Moz: Survey data +The survey results have many common metrics and a few that are different from the correlation data. Here's the set of metrics interesting to me: + +- Page is mobile friendly +- Page load speed +- Page supports HTTPS +- Page is mobile friendly (for desktop results) +- Search keyword match to domain +- Use of responsive design and/or mobile-optimized +- Quality of other sites hosted on the same block of IP Addresses + +Here's a few factors that were considered to negatively impact the ranking: + +- Non mobile friendly +- Slow page speed +- Non mobile friendly (for desktop results) + +In short, mobile friendly / responsive sites are important for both mobile and desktop ranking coupled with site speed. And this makes sense since we're talking about a well designed site that works on both desktop and mobile and loads fast. And it precludes all the content related optimizations! + +## Conclusion + +After looking at the various research and surveys, it is clear that ultimately, Google or other search engines want to provide results that are relevant and popular. For this, they may be using a lot of ranking factors and it may keep changing over time. Trying to address a single ranking factor could be a very hard game. Instead, web masters should work with content creators to provide relevant content that users actually desire. It should be presented in a way that is pleasing and actionable. To aid in this, web maters could: + +- Build mobile friendly websites +- Make it fast - across all the metrics +- Keep it secure and host on a relevant domain +- If needed, associate with a brand so that people and bots trust the page + +All this boils down to Matt Cutts simple statement: +
You never want to do something completely different for googlebot than you'd do for regular users. + +SEOLium blog post
\ No newline at end of file diff --git a/_posts/2017-1-23-how-can-cdn-help-in-your-seo-efforts.md b/_posts/2017-1-23-how-can-cdn-help-in-your-seo-efforts.md new file mode 100644 index 0000000000000..f6976bba99828 --- /dev/null +++ b/_posts/2017-1-23-how-can-cdn-help-in-your-seo-efforts.md @@ -0,0 +1,103 @@ +--- +layout: post +title: How can CDN help in your SEO efforts? +comment: true +--- + +CDN can help in more than just improving site-speed for SEO. It can aid in better targeting, mobile friendliness, domain authority and more. Read about where CDNs are of use for your SEO efforts. + +![standing on shoulders](/images/blog/standing_on_shoulders.jpg) + +## Background +This blog is a follow up for my earlier post on [What metrics matter for SEO?](https://akshayranganath.github.io/what-metrics-matter-for-seo). In this post, I'd like to explore how a CDN can aid in different aspects of SEO. + +But, before we dig in, let's me quickly re-iterate the value proposition of using a CDN. +## Why use a CDN at all? +In the seminal study titled [It's the latency stupid](http://www.stuartcheshire.org/rants/latency.html), Stuart Cheshire tried to understand on which factor matters more for a "faster" website. Is it the bandwidth available or the latency? His conclusion was that beyond a point, bandwidth has no impact on the speed and it boils down to the latency. + +So why is latency such a big speed killer? Simply put, the speed of data transfer over the internet is constrained by the distance from the user and the server. The best possible speed is at the speed of light. However, network components add some processing time and navigating the internet when distances are large means the speed is reduced. + +CDNs deploy their servers such that end users talk to a server that is geographically and topographically closer to them. This way, the network hops and the network think time is reduced. This results in a website speeding up. The basic premise is that the user need only talk to the CDN server and not with the origin data center. So a user in Sydney, Australia will need to talk to a CDN server in Sydney instead of going all the way to a data center in New Jersey, USA. Anecdotally, this all makes sense. + +CDNs also add better routing than the regular Internet's routing and thus improve on the latency. + +With this quick primer, let's dig in to CDN and SEO! +## Role of CDN in SEO +I'd like to discuss the following aspects of SEO. These are the areas where a CDN could help. I've pulled up these SEO factors from Moz's report on [Ranking factors](https://moz.com/search-ranking-factors/). + +- Server response time +- HTTPS / Secure sites +- Domain authority +- Mobile friendly website +- URL Optimization +- (Indirectly) Quality of other sites hosted on the same block of IP Addresses + +### Server response times +This metric is generally translated as TTFB in most studies that have looked at correlation data. However a CDN can help not just in improving the TTFB but, also in reducing the overall latency and thus improving other metrics like page load, start render or speed index. +#### Caching +The most economical and simple solution is to cache the page at the CDN. This should be sufficient as an immediate step towards a better SEO. If our mate in Sydney has to only talk to the CDN server in her city alone, it would be much faster than a request traveling to the server in New Jersey! Caching itself is more nuanced. An object could be cached at multiple places as explained in this post titled [Yoav Weiss](https://calendar.perfplanet.com/2016/a-tale-of-four-caches/">A tale of 4 cachesby announced that it would not penalize users for the HTTP to HTTPS redirect. Setting up HTTPS is easier with a CDN. It is cost effective as well since Certificate authorities like hard. The primary challenge is to identify if a device is mobile/tablet or desktop. This could be avoided by building a purely responsive website where everybody gets the exact same set of resources and the browser displayes according to the device capabilities. However, it causes the most amount of resource bloat for the smallest of the devices. For more on issues around the responsive design, read Guy Podjarny's blog post Domain Authority is a score (on a 100-point scale) developed by Moz that predicts how well a website will rank on search engines. +[Moz: Domain Authority](https://moz.com/learn/seo/domain-authority) + +##### What's the issue +Basically, if you are amazon.com, you just get ranked higher than mom&pop.com. It's because Google has seen amazon.com being delivering results that are relevant and popular. It has a brand that is trusted. Hence Google rewards it by giving it a higher domain authority. + +Other factors that goes into this authority is the use of relevant name. For example, a website about "Web Analytics" named webanalyticsexplained.com will rank better than if it were named as johndoesblog.com. Similarly, domain names that have hyphens and numbers are considered to have lower trust rating and marked down. + +However, domains are hard to setup and managing them would involve effort and time. Case point would be an organization hosting a big event. Suppose www.mycompany.com is hosting a super famous SEO conference called "Best SEO Meet Ever (BSME)". The IT team would find it easier to simply re-use the existing data center and existing firewall rules. In such a case, hosting the conference site on a new domain "www.bsme.com" may be get complicated. +##### CDN To the rescue +With a CDN, domains can be spun up and brought down while the origin data center details remains unchanged. So the CDN could be told to send requests for www.bsme.com to the parent site on a special path like www.mycompany.com/bsme/. Once the event is over, the CDN could even setup a 301 redirect to the parent site so that the audience earned are not lost. + +Bottom line: It is possible to target keyword with a domain and setup it up on a CDN than trying to do the entire setup at origin data-center. +### Other CDN optimizations +#### Handling Failures +When a site suffers outage and Bots try to index and receive error, they are confused. With a CDN it would be possible to setup fail-over mechanisms. Origin failures that are temporary could be coded to respond with a 500. If a maintenance is planned, the CDN can be coded to respond with a 503 and a "Retry-After" header. In call cases, CDN could respond with a simple HTML message that explains the issue as well so that the real users are not left in confusion. + +This ensures bots get the right message and don't index the wrong page or ignore a site for longer than necessary. +#### Stale pages / Redirects +When websites change, they leave behind a legacy of 404s. They are bad for users and are a missed opportunity for SEO. Using CDN, these 404s can be corrected to respond with a redirect to the updated content. This is both a good user behavior and a way of retaining the link juice. + +Do note that redirects have specific meaning with respect to SEO. Have a look at this blog post for more details. +#### A/B, Multivariate testing +CDNs being proxies can act as a control point for your multivariate testing. At the CDN you could definte the logic like sending 10% of mobile traffic to a new site design and then tracking them with analytics or RUM to measure the success criteria. +### Conclusion +In this post, I've tried to cover the reason why a CDN can help in SEO efforts. Apart from improving the latency and bumping up the site speed, CDNs could help in addressing issues of domain authority, managing vanity URLs and targeting efforts. + +If you'd like to know more, DM me [@rakshay](https://twitter.com/rakshay). \ No newline at end of file diff --git a/_posts/2017-1-4-webperformance-notes-from-perfplanet.md b/_posts/2017-1-4-webperformance-notes-from-perfplanet.md new file mode 100644 index 0000000000000..a32ed84a06e30 --- /dev/null +++ b/_posts/2017-1-4-webperformance-notes-from-perfplanet.md @@ -0,0 +1,110 @@ +--- +layout: post +title: WebPerformance notes from PerfPlanet +comment: true +--- + +Every year, in the month of December, [calendar.perfplanet.com](http://calendar.perfplanet.com) invites the experts from Web Performance to contribute their ideas as one blog post a day. It has some very insightful articles and hints at the upcoming technologies. + +![PerfPlanet logo](/images/blog/perfplanet_logo.png) + +I went through the articles and made some notes and thought of sharing it for myself and for anyone who is harried for time. + +### Day 1: [Testing with Realistic Networking Conditions](http://calendar.perfplanet.com/2016/testing-with-realistic-networking-conditions/) + +The routes and peering relationships for all of the CDN’s and servers involved in any given content means that it usually works best if you test from a location close to the physical location your users will be coming from and then use traffic-shaping to model the link conditions that you want to test. + +In the real world if you over-shard your content and deliver over lots of parallel connections with a slow underlying network you can easily get into a situation where the server thinks data queued in buffers had been lost and re-transmits the data causing duplicate data to consume the little available bandwidth. + +### Day 2: [Lighthouse performance tool](http://calendar.perfplanet.com/2016/lighthouse-lighting-to-way-to-better-web-performance/) + +Although lot of tooling is for PWA, it has a command line option and the report is quite useful. Need to try it out. + +**Edit: 26-Mar-2017**: Lighthouse is now built into WebPage test Chrome tests. So keep an eye out for this, + +### Day 3: [Brotli compression](http://calendar.perfplanet.com/2016/enabling-brotli-even-on-cdns-that-dont-support-it-yet/) + +- Brotli over HTTPS only +- FF, Chrome and Opera only + +### Day 4: [HTTP/2 Push - Everything about Push!](http://calendar.perfplanet.com/2016/http2-push-the-details/) +Rules of thumb for H2 push: [https://docs.google.com/document/d/1K0NykTXBbbbTlv60t5MyJvXjqKGsCVNYHyLEXIxYMv0/edit](https://docs.google.com/document/d/1K0NykTXBbbbTlv60t5MyJvXjqKGsCVNYHyLEXIxYMv0/edit) + +Pushing resource 4 use cases: + +- after html +- before html (most interesting, works with CDNs) +- with resources +- during interactive / after onload + +Resource hints are cross origin while H2 push is not. + +Lots of challenges with push, esp Push+Resource hints. Performance varies between cold and warm connection and degrades with higher latency connections like 2G. Lot more research required. + +Colin Bendall's [https://canipush.com](https://canipush.com) is a good resource to test browser capability to accept server side push. + +### Day 5: [Meet the web worldwide](http://calendar.perfplanet.com/2016/meet-the-web-worldwide/) + +Get data on different aspects of web usage from: https://www.webworldwide.io/ +Desktop + +### Day 6: [Measuring WebPageTest Precision](http://calendar.perfplanet.com/2016/meet-the-web-worldwide/) + +For a desktop experience, the default 9 runs yielded following precision: + +- TTFB: around 6% to 8% +- Other metrics: better than 6% +- If 3% precision or better is sought, then 20 or more runs are recommended. +- If a 10% precision only is sought, then 7 runs should be enough. + +Mobile + +- For a mobile experience, the default 9 runs yielded a 3% or better precision. +- If a 5% precision only is sought, then 5 or more runs should be enough. +- If a 10% precision only is sought, then a single run should do. + +### Day 7: [Progressive Storyboards](http://calendar.perfplanet.com/2016/progressive-storyboards/) +It’s important to ensure that the sequence in which features are revealed to the user is natural and follows user’s incremental needs as they wait comfortably for all the information and interactive functionality to be shown. + +1. Verify Destination + * First step of any web navigation from one page to another or interaction within website interface is to assure the user that the action they performed was the one they intended to do so they can comfortably wait while it loads without wondering if they need to click the Back button. +2. Provide primary content +3. Allow interation +4. Show secondary content +5. Below the fold + +### Day 12: [Prefer DEFER over ASYNC](http://calendar.perfplanet.com/2016/prefer-defer-over-async/) +Async will not block HTML parsing but it will block rendering. So prefer DEFER over ASYNC when possible. + +### Day 17: [Rise of Web Workers](http://calendar.perfplanet.com/2016/rise-of-the-web-workers/) +Web workers may be used to handle all the non-DOM manipulation tasks, especially when being used in a frame-work based environment like React and Angular. Needs more research though. + +### Day 20: [Font-face syntax optimizations](http://calendar.perfplanet.com/2016/no-font-face-bulletproof-syntax/) +Remember that browsers use the first format they find that works—so if you don’t order them correctly the browser could waste resources (or render poorly) with a less-than optimal format. + + @font-face { + font-family: Open Sans; + src: url(opensans.woff2) format('woff2'), + url(opensans.woff) format('woff'); + } + +First use woff2 and then woff. Doing this eliminates older browsers but, they'll still see the content on system defined fints. + +### Day 24: [A tale of 4 caches](http://calendar.perfplanet.com/2016/a-tale-of-four-caches/) +There are different levels of caching on the browser and the behavior may vary based on the way it was loaded. eg: preloaded object may not persist across navigation as compared to a prefetched object. The location of cache also has implication on whether an object shows up in the Developer tools. + +### Day 25: [Root Domain issues with CDN](http://calendar.perfplanet.com/2016/root-domain-cdn-performance/) +The ANAME/ALIAS is resolved by your DNS provider’s nameserver instead of by a recursive resolver (ISP, Google Public DNS, OpenDNS or other) and this may lead to end users being routed to a far-away CDN node and consequently getting a poor experience. + +### Day 26: [PNG Image optimizations](http://calendar.perfplanet.com/2016/squeezing-png-images/) +PNG optimization using pngquant and zopflipng + +### Day 27: [HTTP Push and Progressing JPEGS](http://calendar.perfplanet.com/2016/even-faster-images-using-http2-and-progressive-jpegs/) +Progressive images with HTTP/2 should help in renderimg images faster. To optimize it further, consider changing the scan settings at the image optimization work-flow stage to improve on the SpeedIndex. + +### Day 28: [Links to interesting posts](http://calendar.perfplanet.com/2016/bonus-content/) + +There are a lot of articles mentioned in here. I am going to focus on these 2 for now: + +* [R for website performance optimization](https://github.com/craigfrancis/r-stats/blob/master/docs/intro.md) +* [2017 front-end performance checklist](https://www.smashingmagazine.com/2016/12/front-end-performance-checklist-2017-pdf-pages/) diff --git a/_posts/2017-11-07-3-Reasons-Why-AMP-And-CDN-Can-Work-Together.md b/_posts/2017-11-07-3-Reasons-Why-AMP-And-CDN-Can-Work-Together.md new file mode 100644 index 0000000000000..61c80bd594f68 --- /dev/null +++ b/_posts/2017-11-07-3-Reasons-Why-AMP-And-CDN-Can-Work-Together.md @@ -0,0 +1,96 @@ +--- +layout: post +title: 3 Reasons Why AMP And CDN Can work together +comment: true +description: Google's Accelerated Mobile Pages (AMP) and Content Delivery Networks (CDN) are complimentary. In this post I'd like to show that the two can co-exist. +image: /images/blog/fist-pump.jpg +tags: [amp, cdn] +--- +Google's Accelerated Mobile Pages (AMP) and Content Delivery Networks (CDN) are complimentary. In this post I'd like to show that the two can co-exist. + +## What is AMP? + +According to Google [Accelerated Mobile Pages (AMP)](https://developers.google.com/amp/) project +>is built on top of existing web technologies to enable blazing-fast page rendering and content delivery. + +Although Google's stated purpose is about speed, the adoption has been due to the carrot it danlges for AMP users. A page that implements AMP could be shown as part of the carousal right under the search bar. A non-AMP page will never be shown here. This SEO jump is what has been driving the folks to try AMP. To get into this carousal, you need to have your content in Google's AMP cache. But - we're getting ahead of oursleves. Let's see how AMP works. + +## How does AMP work? + +AMP consists of 3 parts. Again, taken from the AMP project: + +- _AMP HTML_ is HTML with some restrictions for reliable performance. +- _AMP JS_ library ensures the fast rendering of AMP HTML pages. +- _AMP Cache_ can be used to serve cached AMP HTML pages. +If your page implements the AMP HTML, passes the validation by AMP JS, then you are eligible for AMP CDN caching. Elaborating on the caching, Google explains: +>The Google AMP Cache is a proxy-based content delivery network for delivering all valid AMP documents. It fetches AMP HTML pages, caches them, and improves page performance automatically. When using the Google AMP Cache, the document, all JS files and all images load from the same origin that is using HTTP 2.0 for maximum efficiency. + + +An immediate question that comes to mind is: _Do we remove CDN since we now have AMP and save some money?_ Well, the answer is you will still need a CDN and here are 3 reasons for it. + +![team work](/images/blog/fist-pump.jpg) + +## Reason 1: AMP pages still need to be hosted + +AMP pages are discovered like regular pages by Google. So you need to have a webpage that is hosted on the internet. Here's a the work-flow that Google uses to discover and cache an AMP page. For details and an example, refer to the [case study](#case-study) below. + +![How is AMP Page Discovered?](/images/blog/how-is-amp-page-discovered.png) + +Google detects that an AMP version exists by looking for the ```<link rel="amphtml"``` tag. Content publishers will potentially need to create 2 versions of a page and host it at origin. Since these are pages that can be cached, serving them over CDN will ensure better performance and a good offload at the origin data center. + +## Reason 2: Google still needs to index + +Google claims that _site speed_ is one of the indicators for ranking a web page. To get to a good performance, you'd need to leverage a CDN. This will ensure that the latencies are reduced for the Google bot, regardless of the bot's location and your site has a good performance and better chance of ranking well on Google. + +## Reason 3: AMP is not universally supported + +The target for AMP project was mainly publisers. These are the pages that can easily be cached and has little to no personalization. If you are running an website like an online shop with personalized content, A/B tests and device specific optimizations then, AMP will not be a good fit. + +AMP is also not designed for pages that have dynamic elements. So if you have a form or a check-out flow, AMP will not work for you. In all these cases, you'd need to host a version of your website and use a CDN to deliver and optimal experience to your users. + +## Case Study +With the basic story out of the way, I wanted to show a real use case. Let us take the example of [Forbes](https://www.forbes.com/). Let's use an example of a news story. + +- News Story URL: [https://www.forbes.com/sites/bobevans1/2017/10/18/ibm-rocks-the-cloud-purists-moan-but-customers-love-big-blues-15-8-billion-cloud-business/#60b5986e46e3](https://www.forbes.com/sites/bobevans1/2017/10/18/ibm-rocks-the-cloud-purists-moan-but-customers-love-big-blues-15-8-billion-cloud-business/#60b5986e46e3) + +Now, this page is the full user experience. If you look at the source code, it has a hint to the AMP page using this tag: + +```html + +``` + +Let's follow the link and open the AMP page. Within this AMP page at the URL [https://www.forbes.com/sites/bobevans1/2017/10/18/ibm-rocks-the-cloud-purists-moan-but-customers-love-big-blues-15-8-billion-cloud-business/amp/](https://www.forbes.com/sites/bobevans1/2017/10/18/ibm-rocks-the-cloud-purists-moan-but-customers-love-big-blues-15-8-billion-cloud-business/amp/), there is a reference to the AMP JS. + +```html + +``` +This tells Google that Forbes wants the page to be validated and cached on Google AMP cache. So Google does the validation and saves it at the URL [https://www-forbes-com.cdn.ampproject.org/c/s/www.forbes.com/sites/bobevans1/2017/10/18/ibm-rocks-the-cloud-purists-moan-but-customers-love-big-blues-15-8-billion-cloud-business/amp/](https://www-forbes-com.cdn.ampproject.org/c/s/www.forbes.com/sites/bobevans1/2017/10/18/ibm-rocks-the-cloud-purists-moan-but-customers-love-big-blues-15-8-billion-cloud-business/amp/) + +There is a pattern that Google follows to name the AMP domain. This is explained in the [AMP Cache URL Format](https://developers.google.com/amp/cache/overview#amp-cache-url-format) + +### Hacks Around AMP + +#### Use AMP without AMP cache +Using the JS code to validate your page is the step that actually causes Google to create the Google cache domain and serve the objects from here. If your goal is to build a well-performing web page but without the Google AMP cache, then simply remvove the reference to the JS file. By doing this, you get the advantage of a well performing page and full control over the content. + +The downside is you will not appear in the carousal below search box. + +### Create AMP-only page +You could create a website that has AMP-only version. In this case, the ```<link rel="amphtml"``` tag will point back to the same URL to tell Google that this page is an AMP version. + +For example, [BMW.com](https://www.bmw.com/en/index.html) has a new website that is based on AMP (and progressive web app). If you notice the source code, it starts with this tag: + +```html + +``` +So this page itself will be served for mobile search results (from AMP cache) and from the regular domain for all other uers. + +## Concluding thoughts +AMP is targeted for mobile search results and for faster rendering of content on the mobile devices, especially in constrainted network conditions. If you are a web-admin who wants to provide a rich user experience and immersive images, then AMP is not necessarily the right approach. It can give you a marginal SEO boost but, at a cost of minimalistic experience. + +Regardless of the approach you use, there still needs to be a web site that is hosted. This website still needs to be fast, reliable and scalable to handle the end user and bot requests. It can either provide a minimalistic AMP experience or a rich experience. For this part of the website, you would require a CDN and hence the argument that AMP and CDN work together. + + + + + diff --git a/_posts/2017-11-15-Notes-From-Enterprise-DevOps-Summit-San-Francisco.md b/_posts/2017-11-15-Notes-From-Enterprise-DevOps-Summit-San-Francisco.md new file mode 100644 index 0000000000000..e1bc3c643e2bf --- /dev/null +++ b/_posts/2017-11-15-Notes-From-Enterprise-DevOps-Summit-San-Francisco.md @@ -0,0 +1,516 @@ +--- +layout: post +title: Notes from Enterprise DevOps Summit 2017 - San Francisco +comment: true +description: I was privileged to be sponsored by Akamai to attend the Enterprise DevOps Summit in San Francisco. Here are the notes from the sessions I attended. +image: /images/blog/DOES2017SFO.png +tags: [devops, conferences, san francisco] +--- + +I was privileged to be sponsored by Akamai to attend the Enterprise DevOps Summit in San Francisco. Here are the notes from the sessions I attended. + +![DOES 2017 SFO](/images/blog/DOES2017SFO.png) + +All the resources for the conferences were already shared. Here are the links: + +- [Presentations on Github](https://github.com/devopsenterprise/2017-San-Francisco) +- [Presentations on DropBox](https://bit.ly/DOES17SFOslides) +- [Videos on Youtube](https://www.youtube.com/channel/UCkAQCw5_sIZmj2IkSrNy00A/feed) + + +# Day 1: 11/13/2017 + +## Keynotes + +## More Culture, More Engineering, Less Duct Tape + +Tranformation methodology is very similar to _Improvment Kata_ + +## Augmenting the Org for DevOps + +- significant outage in 2015. Driver and accelerator for DevOps movement +- application had over 200 hops on the internal network for a single interaction - this was the application that had to move to cloud + +Methodology: Monolith to 3 tier app, re-skilling org to Agile processes + +Critical Success factors: +- get CXX support +- metrics based +- changing team structured from contractors to employees => end to end engg rather than narrow focus. + +Manual testing was the primary bottleneck (20 hrs). Automation reduced it to 12 mins. + +Automated regression testing +- grass roots bottom-up approach +- know how to pitch upwards (eg: talk about reduced lead times instead of no of deploys a week) + +**Containergeddon** - everything is moving to containers. + +Used [circuit breakers](https://martinfowler.com/bliki/CircuitBreaker.html) and [Netflix hysterix](https://github.com/Netflix/Hystrix) code + +DevOps value: +During an outage, 10 UI changes were released in 3 days. Even the CEO made statements on the success of DevOps movement. + + +## Using DevOps to Build Your Learning Organization +[@stevemayner](https://twitter.com/stevemayner) + +- Columbia Sportswear - Family owned business. +- DevOps movement was implemented without a CXX sponsor / support. +- DevOps born out of the need to stay relevant and to "do the right thing" + +Challenge: Microsoft shop with vendor lock-in. Very UI driven and lack of automation. + +> You are either building a learning organization or losing to someone who is. - _Andrey Clay Shaffer_ + +_Be -> Know -> Do_ model of leadership + +- Version control was a big game changer +- Go for small wins using the Plan-Do-Check-Act (PDCA) model. + +Referenced a velocity talk about Ops in one of the roles: +- rockstars +- builders +- janitors + +[Watch this](https://www.youtube.com/watch?v=posb7CzWSFc) + +_"Weekly retrospective"_ added a huge value and gave a lot of insight on how to improve from grass-roots. + +_ChatOps_ - game changer + +**Anti-pattern** to most others: If some task is not common, then automate it. Since it is not repeated often, potential to make an error is high. + +## Transformational Leadership and DevOps - Beyond the Research + +Cultural change is the hardest change. Success rate of just 19%. But, ability to change is the core competency of organizations. + +Why do transformations fail? +Confusion -> Frustration -> Anxiety -> Fear (now people are at the lowest tier of Maslow's pyramid) -> Resistance + +![Transformation Leadership model](https://techbeacon.com/sites/default/files/styles/inline_image/public/maynerfigure_0.png?itok=E0nRDj1l) + +Books referenced: +- The Startup Way +- Turn the ship around + +Middle managers are the ones most impacted by change. However, +- They know the processes, +- have the network and +- can point to places where skeletons are buried. +This is very important in transformation so that mistakes aren't repeated. + + +## Breakout sessions + +### What Does it Mean to Lead IT? +**[Mark Schwartz](https://twitter.com/schwartz_cio?lang=en)** + +Historically, IT department was an "add-on". Business went on fine w/o IT. Streotypes created for IT: +- uninterested in business +- don't care about timelines +- can't even speak in English (as in can't speak in terms understood by others) + +So, IT had to be provided with exact requirements. IT had to respond with goals, timelines and budget. Business would control IT and extract value. IT was never considered an integral part of the org. ==> Waterfall model. + +"IT's" customer is business mentality. As though, IT did not contribute anything to the actual product. + +CIO's office created to: +- talk English to business +- come up with measurable goals, even if Biz did not understand what the goals meant +- enforce timelines and budget + +No other department in the org is "controlled". They have freedom to operate and do the necessary things to help business succeed. Not IT. + +Even with _Agile_ and _DevOps_, the attitude hasn't changed much. It is about business controlling IT. + +#### New model +Instead of requirements, provide high level objectives. Let IT be a part of the strategy discussion. Since teams are already multi-functional, no need to control. + +Role of the leadership: +- provide clear objectives +- remove impediments +- monitor the progress towards business objectives + + +### Continuous Chaos in DevOps + +Build chaos engineering tools as a foundational service. Every application has to "subscribe" to the tools and specify the components. This will identify the right failure services to use. "Failure as a service" model. + +### DevOps Handboook Experiments in Accelerating Delivery + +Lessons implemented from the DevOps hand book: +![DevOps Handbook lessons](/images/blog/DevOps%20Handbook%20lessons.png) + +Instead of "presenting" to management, rely on "Show and Tell". It provides better grasp to management. + +One of main benefits of DevOps - automation testing reduced the lead time. + +### Architecting Your App And Your Pipeline for Continuous Delivery - 10 DOs for Successful DevOps +[@anders_wallgren](https://twitter.com/anders_wallgren) + +One of the best predictors of high performing organizations - "how much do they fear deployments"? + +Another one: Binary artifact repo - indicates maturity for devOps success. + +Very technical deck. Extremely useful for development teams / architects. Speaks about software patterns to build code and for the pipelines. + +Some good resources from [microservices.io](https://microservices.io) + +Follow the [@honest_update](https://twitter.com/honest_update?lang=en) for some fun status messages. + + +## Day 2: 11/14/2017 + +### Keynotes + +#### Better Governance - Banking on Continuous Delivery + +Goals for 2017 and beyond: +- slayTheMonolith +- noFearRelease +- youBuildItYouOwnIt + +##### NoFearRelease +Typical fears are: +1. fear of speed +2. fear of breakdown +3. fear of being out of control +4. fear of being non-compliant + +_Compliance_ is a bad word. _Governance_ is better. It implies _awareness and ownership of risk_. + +3 lines of defense: +- 1st line: Who owns the risk? +- 2nd line: policies and processes +- 3rd line: independent assurance + +Minimum set of controls: +- Two sets of eyes (peer review +- Least privilege +- Unauthorized change monitoring + +Being bank, the question from auditors were: How to assure that the pipeline itself is not tampered, the process is not broken or the pre-commit checks were not relaxed for a driven deployment. + +In terms of deployments, Kubernetes / containers provide immutable servers. And no human access is provided to production systems. + +Opensource tools: [CapitalOne](https://github.com/capitalone/) + + +#### The Making of Amazon Prime Now + +![amazon virtuous cycle](/images/blog/amazon-maturity-model.png) + +_i had to work so could not focus much_ + +#### Making Digital Magic + +Digital transformation involved: +- Technology +- Leadership +- Community +_very similar to the People-Process-Technology model in Agile_ + +Mesos - what is it? + +Technology for _Movies Anywhere_ +- cloud native +- ops deeply embedded into design +- Kubernetes with Kops +- Terraform +- helm, vault +- Lambda functions used for monitoring +- PagerDuty, NewRelic and Data dog for monitoring + +Reference to book - [What Got You Here Won’t Get You There](https://www.amazon.com/What-Got-Here-Wont-There-ebook/dp/B000Q9J128/ref=pd_sim_351_1?_encoding=UTF8&psc=1&refRID=9DE6NZ7X0C0RDE4N5214) + +### Breakouts + +#### Intel's Journey to Build Quality In: How QA and Test Automation Drive DevOps Transformation + +Initial model _push and pray_. 85% of code was not covered. + +Parameterization of tests and using environment variables made a big difference in the usabilty of tests. + +On-demand is the key. Environment needs to be spun up and down with no tickets and wait times. + +#### Cloud Native Architectures +**[Cornelia Davis](https://twitter.com/cdavisafc))** + +Just download the presentation. It is simply too good. Try to get the book as well. +[Slides](https://www.slideshare.net/cdavisafc/cloud-native-architectures-for-devops) + + +#### Starting and Scaling DevOps in the Enterprise + +Very informative and down to earth presentation on DevOps transformation. Download slides for future use. + +CAB is the "change resistance board"! + +Today's approach to DevOps: +> I am a millenial. My mom still cleans my room and ops will clean up after me. + +In today's deployments, more time is taken to setup the environment and run the checks than the actual coding. + +Repeated manual tests: (3 Ms ) + +- waste of time (Muda) +- inconsitancy (Mura) +- burden on testing team (Muri) + +Book - Jeff Morgan on testing + +Action: Sign up for newsletter and get the book free. + +#### DevOps: From Analyst Inquiry to Organizational Action + +**Trends** + +- Last year questions were about - "What is DevOps"? +- This year, DevOps has reached escape velocity. +- Trends for this year: + - Dev and Ops are now working together. + - Security needs to fit in. + - Data / database is still outside of the scope. This will increasingly become the bottleneck and requires newer architectures. +- In last 90 days, Business is coming back and saying that deployment are too fast. Net result: Deployment frequency is being re-defined as "on demand deployments" +- **Technologies**: Gitlab, Jenkins, Kubernetes and Jira - 80% of organizations use them. However, what's missing is end-to-end tool chains. +- Insurance, Banking and FinTech - DevOps is _smoking hot_. + +Again - download slide for future use. + +#### DevSecOps: It's Not Me or You, It's WE! + +Panel discussion.. + +Security is traditionally "command and control". Sec teams tend to say - "here's the code / process. Just follow it." No longer the right approach. + +Security companies still stuck in last century of request/response or client/server model. + +Follow NIST security framework instead of ISO. ISO is too burdensome. + +_Approach for security_: + +- what security incidents were handled +- what security incidents were failures +- Pen Test and take action items and fix the bugs. + +**Closing tips**: + +1. Break silos between security and devops teams +2. Practice often: War games, incident response +3. Security should be "built-in". It should be part of the product design and part of the build pipeline. Security team needs to understand the business objectives. +4. Know your bill of materials, i.e., Configuration Items should be clearly known. This will help address issues like struts vulnerability. + +Finally, aim for "security as code". Although not currently a feasible solution, this is the end goal. + +#### The Key to High Performance: What Data Says +[@nicolefv](https://twitter.com/nicolefv) + +All about the **State of DevOps** reports. Trends: +- 2014: DevOps Works! +- 2015: IT goes lean +- 2016: Shift left +- 2017: DevOps works for all kinds of organizations. + +Other notes: +- Measure **Cycle time**: code commit to code deployment. +- Throughput and stability are possible without any trade offs. +- Maturity models don't work in this industry. "What is a mature org?" - metrics itself keep changing very fst. +- Architecture matters - technology stack doesn't +- Westrum's organizational model and transformational leadership - very good predictors of DevOps successes. + +Read the HBR article by _Besson_ on _[Real Reason Superstar firms are racing ahead](https://hbr.org/2017/10/the-real-reason-superstar-firms-are-pulling-ahead)_. + +#### Coping With Complexity: Resilience Engineering Research In DevOps + +Action item: Download and share the [stellar report](http://stella.report/). + +> incidents of yesterday inform and the architecture and rule of tomorrow. + +![incidents as investment](/images/blog/allspaw-incidents.png) + +Check out [adaptivecapatictylabs.com](https://adaptivecapatictylabs.com/) + +Watch the video again. It is amazing! + +## Day 3: 11/15/2017 + +### Keynotes + +#### The human factor: Inspiring the pursuit of success and averting drift into failure + +Session by [Dr Sidney Dekker](http://sidneydekker.com/) + +- the company / airline that reports the most incidents has the least number of catastrophic accidents ==> open culture and less fear of being honest +- sweet spot of honesty exists for organizations.. if you stress beyond it, catastrophic results occur +- award for speaking up is generally unclear and delayed. award for _not_ speaking up is generally immediate + +_What's difference between organizations that screw up and those that don't screw up?_ +It is the presence of positive capacities and not in the absence of negative capacities. +_This follows John Allspaw's talk. When trying to find the root cause, analyze why the incident had not occurred more times - what causes the lack of incidents._ + +Understand how things go right to figure out processes on when things go wrong. + +![Dekeer - focus on what's right](/images/blog/dekker-focus-on-right.png) + +#### Fear does not exist in the dojo - a devops journey with a competitive twist + +_DoJo_ - creating an internal training environment. +Process cross-pollinated to Verizon from Target. + +6 weeks workshop: +- 2 day Sprint +- 12 sprints +- Clear outcomes of the effort +- Up skilling of employees to new ways of working, esp in global org +Note to self: Can our consulting work-shop be modeled on this? + +Internal DevOpsDays - a very good way to engage, encourage and learn. + +![Internal devops conference](/images/blog/devops-internal-conference.png) +Gamification of DevOps and get teams excited. + + +#### There is No Finish Line + +No "devOps" team - everyone has to adopt it. + +![nike transformation](/images/blog/nike-transformation.png) + +### Breakouts + +#### The Case for Value Stream Architecture + +Competition is not across the board. It can be at specific value stream. Case point - +![Unbundling of banks](/images/blog/Unbundling-of-a-bank-V2.png) + +DevOps is big and there are a lot of startups +![DevOps startups](/images/blog/devops-and-programmatic-infrastructure-37-638.jpg) + +DevOps / Lean tries to take the analogy of assembly line. Although accurate to some extent, it does not capture the complexities of software developement: +- Car assembly lines can stay stable for about a year. Software keeps changing. +- Car manufacturing can follow linear steps. Software release is not all that linear. +- Software value stream looks more like an airline network with some hubs seeing more dense connections than others +- Replace airports by tools and process - this is a more accurate representation + +Waterfall and very narrowly focused DevOps movement is trying to force linearity where linearity doesn't really exist. + +Sample work-flow at Nationwide: +![CI visibility at Nationwide](/images/blog/how-nationwide-and-tasktop-achieved-continuous-visibility-across-the-devops-lifecycle-7-638.jpg) + +Download _Value stream architecture_ eBook + +Download book - [Thinking Environments](https://itrevolution.com/book/thinking-environments/) + + +#### Enterprise DevOps and Unicorns + +Fear / change approval is a mind killer. + +#### Process-as-Code: Real-World Examples that Scale + +Treat your automation process as a product / software artifact. So it goes through rigorous testing, validation, enhacements and bug fixes. + +Follow-up: +- what is groovy +- what is Domain Specific Language (DSL) +- what is vagrant + + +#### Baking Security into your Pipeline: Start Here + +Embed security experts in SCRUM meetings. + +Hacked pipeline is very scary +- when plugins are used in Jenkins, who checks for vulnerabilities in them? + +Credentials should be in [vault](https://www.vaultproject.io/) and not hard-coded or in plain-text configuration files. + +Work on reducing _blast radius_ by reducing the impact of a compromise. + +What is SonarQube? + +*80% of official (latest) docker images had at least 1 severe vulnerability* + +Configuration item + auto patch + +Keep testing your process. Process as code means treat it as: +- version control +- test +- log +- audit trail + + +#### Microservices for the Enterprise: Myths vs. Reality + +This session seemed to be put together without any real agenda or focus. Very boring :-( + +#### Convergence of Safety Culture and Lean: Lessons from the Leaders + +_Panel Discussion_ between Gene Kim, Dr Sidney Dekker, Dr Steven Spear and Dr Richard Cook. + +Leaders -> enable team. Not control it. Leaders need to say: +- I don't know +- I need help +If this is not seen by org, then learning culture will never take effect. + +Most important learning from Toyota Production System - Andon Chord ==> recognize that the known processes is not able to cope up with current work processes. + +##### Guidelines to Community +- Tell stories, esp when things go wrong. It helps sensitize community on what can go wrong. +- DevOps is about building community of users who work on devOps. Mentor others. +- Share stores of failures and near misses.. _Success comes from experience and experience comes from failures_. +- You are all experts and you have moral responsibility to share it with the community. +- Identify yourself as one who practices DevOps instead of identifying as someone working for a particular role in a company +- Share stories of uncertainty + + +### Closing Keynotes + +#### Data and DevOps: Breaking Down the Silos +**[Elisabeth Hendrickson](https://twitter.com/testobsessed) + +Data is still in silos. +1. DBA +2. Data Engineer +3. Data Scientist + +Read up on [12 factor app](https://12factor.net/) + +Cloud Native architecture for data. +![cloud native architecture](/images/blog/cloud-native-architecture.png) + + +#### The Yin and Yang of Speed and Control +**[Jon Smart](https://twitter.com/jonsmart) + +Read up on [Cynefin framework](https://en.wikipedia.org/wiki/Cynefin_framework) + +Lessons at Barclays bank: +1. Don't scale work. Create smaller teams and batches first. +2. Don't impose an Agile transformation. Focus on outcomes - start with why + +>It is easier to act your way to a new way of thinking rather than think your way to a new way of acting. + +>Impediments is not in the path but impediments is the path. + +Every requirement should follow a Hypothesis Drive Development by [@barryoreilly](https://twitter.com/barryoreilly) +![Hypothesis driven investment](https://insights-images.thoughtworks.com/HDD_StoryCard_10a9a864fa5f5db4bbaea5bb39dddf60.jpg) + + +### Books referenced + +- The Startup Way +- Turn the ship around +- What Got You Here Won’t Get You There +- Cucumber and Cheese - A Tester's Workshop +- [Cloud Native](https://www.manning.com/books/cloud-native) + +### Key trends + +1. DevOps is in full-swing accross industries +2. Cultural transformation is the hardest but, it is necessary. +3. Focus is shifting from plain "deployment frequency" to being ready for "on-demand" deployments. +4. Security, esp security of code and the pipeline is gaining attention. +5. Maturity in-terms of expanding the DevOps to mean more teams like audit, marketing, finance and legal teams +6. Data is the hardest problem to move to cloud based architecture. This is going be to an ongoing item for investigation. \ No newline at end of file diff --git a/_posts/2017-12-01-DevOps-A-Holistic-View.md b/_posts/2017-12-01-DevOps-A-Holistic-View.md new file mode 100644 index 0000000000000..21c1591aa7b73 --- /dev/null +++ b/_posts/2017-12-01-DevOps-A-Holistic-View.md @@ -0,0 +1,238 @@ +--- +layout: post +title: DevOps - A Holistic View +comment: true +description: DevOps is influenced by many ideas like Lean, ToC, Agile and ITIL. In this post, I point out the exact practices adopted from these methods. +image: /images/blog/collaboration.jpg +tags: [devops, lean, agile, theory of constraints, ITIL] +--- + +DevOps is influenced by many ideas like Lean, ToC, Agile and ITIL. In this post, I wanted to explore the origin of different practices to give you a holistic view. The reason is that most blogs or articles just allude to this by saying "DevOps is based on Lean, Agile, ITIL and other practices" without explaining what part of the DevOps methodology is derived from these earlier methodologies. + +Hope this journey into the origins of DevOps is exciting! + +## What is DevOps? + +DevOps is a movement and not necessarily a process. It has various definitions by different authors. Here is one that I feel does a decent job. All the **emphasis** are mine. + +>DevOps represents a change in IT **culture**, focusing on rapid IT service delivery through the adoption of agile, lean practices in the context of a **system-oriented** approach. DevOps emphasizes **people** (and **culture**), and seeks to improve **collaboration** between operations and development teams. DevOps implementations utilize technology — especially automation tools that can leverage an increasingly programmable and dynamic infrastructure from a life cycle perspective. [Source: [Gartner on DevOps](https://www.gartner.com/it-glossary/devops)] + +So let's break this down a bit: +* DevOps is about people and culture +* It is about a _system-oriented_ approach. Whenever you see the term _system-oriented_ it just means the focus is on the business outcome. e.g.: A systems-oriented goal for an eCommerce website is more revenue. For a Dev team, it could be faster code release and for an Ops team, it is the no of days without an incident. The idea is that ultimately, business goal is everyone's goal and optimization has to be made towards this goal rather than team specific ones. +* Tools are not the reason for DevOps - they only enable in the DevOps culture + + +![DevOps Balanced Scorecard](/images/blog/DevOps-Strategy-Map-Draft-v4-020316.gif) +## Breaking down DevOps + +DevOps as a practice has evolved from various fields. According to the [The DevOps Handbook](https://www.amazon.com/dp/B01M9ASFQ3/ref=dp-kindle-redirect?_encoding=UTF8&btkr=1), it is based on: +* Lean +* Theory of Constraints +* Toyota Production System +* Resilience Engineering +* Learning Organizations +* Safety Culture +* Human Factors +* High-trust management culture +* Servant leadership +* Organizational change management +* .. and many more + +These are a lot of items and it is hard to see which practice of DevOps is influenced by these practices. So let's try to fit them in. I am using [The Three Ways: The Principles Underpinning DevOps](https://itrevolution.com/the-three-ways-principles-underpinning-devops/) as the reference to explain the mapping. So we have: +* The First Way: This talks about the flow of work from left to right. +* The Second Way: This is about the feedback from right to left +* The Third Way: It emphasizes the need for continual experimentation and learning. + +Let's see how the 3 ways work and the basis for these concepts. + +## The First Way +This is the way of describing the flow of work from left to right, ie. from idea to delivery of working software code. At its heart, it just means that if we remove any blockers, we are able to deliver faster. +![First way](/images/blog/first_way.png) + +The First Way requires the fast and smooth flow of work from Development to Operations, to deliver value to customers quickly. We optimize for this global goal instead of local goals, such as Development feature completion rates, test find/fix rations, or Ops availably measures. (_Source: DevOps handbook_) + +The goal of fast flow is to make any kind of waste and hardships - anything that requires heroics is made visible and to systematically alleviate or eliminate the burden. + +### Agile + +Let's first start with the definition of Agile. + +>Agile software development refers to a group of software development methodologies based on iterative development, where requirements and solutions evolve through collaboration between self-organizing cross-functional teams. (Source: [WHAT IS AGILE? WHAT IS SCRUM?](https://www.cprime.com/resources/what-is-agile-what-is-scrum/)) + +![Agile model](/images/blog/software_quality-agile_software_dev_cycle_desktop.jpg) + +The core ideas are: +* self-organizing teams +* smaller batch sizes +* working software in smaller increments + +The idea is business and development is closely aligned and hence able to deliver on the __actual need__ from business. This is in contrast to _waterfall_ processes where the requirements are created right at the beginning and business has no visibility until the product is ready. + +However, the focus of Agile is on Development and does not really explain what occurs when the code is thrown over the wall to Ops. + +This is where DevOps picked off and wants to bring the whole process from creation to delivery and operations into the scope. + +DevOps and Agile can play well. For example, a user story is marked as _Done_ only after it is deployed instead of just the cod check-in. Similarly, Ops can be emedded in the daily SCRUM meetings to ensure environments are created and ready for use when required. + +### Lean + +According to [Wikipedia](https://en.wikipedia.org/wiki/Lean_manufacturing) + +>Lean manufacturing or lean production, often simply "lean", is a systematic method for waste minimization ("Muda") within a manufacturing system without sacrificing productivity. Lean also takes into account waste created through overburden ("Muri") and waste created through unevenness in work loads ("Mura"). Working from the perspective of the client who consumes a product or service, "value" is any action or process that a customer would be willing to pay for. + +![Muda-Mura-Muri](/images/blog/muda-mura-muri.jpg) + +Lean basically originated from the [Toyota Production System (TPS)](http://www.toyota-global.com/company/vision_philosophy/toyota_production_system/) + +The core aspects that impacts DevOps are: +* _Muda_ / waste: Waste in terms of DevOps is defects or broken builds. +* _Mura_ / unevenness: This is manifested in snowflake configurations, mismatched environments. This concept is explained quite well in the presentation [The History of Pets vs. Cattle .. & Using it correctly](https://www.slideshare.net/randybias/the-history-of-pets-vs-cattle-and-using-it-properly) + +From DevOps perspective, the concept of "flow", "value streams" "bottlenecks" and "continuous improvement" derive from Lean. Specifically, Lean manufacturing has a 5 part priciple that is used to guide the production. Here are the 5 steps. + +* Specify value from the standpoint of the end customer by product family. +* Identify all the steps in the value stream for each product family, eliminating whenever possible those steps that do not create value. +* Make the value-creating steps occur in tight sequence so the product will flow smoothly toward the customer. +* As flow is introduced, let customers pull value from the next upstream activity. +* As value is specified, value streams are identified, wasted steps are removed, and flow and pull are introduced, begin the process again and continue it until a state of perfection is reached in which perfect value is created with no waste. +_Source:[PRINCIPLES OF LEAN](https://www.lean.org/WhatsLean/Principles.cfm)_ + +![Principles of lean](/images/blog/5stepslean.gif) + +#### Kanban +One of the core tenets of the _First Way_ is to make the work visible. To achieve this, Kanban boards are used. + +>Kanban (看板) (literally signboard or billboard in Chinese and Japanese) is a scheduling system for lean manufacturing and just-in-time manufacturing (JIT).[2] Kanban is an inventory-control system to control the supply chain. _Source: [Kanban](https://en.wikipedia.org/wiki/Kanban) + +When a Kanban board is used it is very clear on the tasks being queued up, in progress and completed. The end goal is to reduce idle time and eliminate waste. + +![Kanban board](/images/blog/kanban-board-e60650d1-1.jpg) + +### IT Service Management (ITSM) + +Let's begin by understanding the meaning of ITSM and ITIL. + +>IT service management (ITSM) is simply how you manage the delivery of end-to-end IT services to your customers based on best practices. One of the most commonly adopted best practice frameworks for ITSM is ITIL or IT Infrastructure Library. _Source: [What is ITSM?](https://www.atlassian.com/it-unplugged/itsm)_ + +>The Information Technology Infrastructure Library (ITIL) is a set of practices for ITSM that focuses on aligning IT services with business needs. ITIL is the most widely accepted approach to ITSM and can help IT organizations realize business change, transformation, and growth. _Source: [What is ITSM?](https://www.atlassian.com/it-unplugged/itsm)_ + +ITSM is similar to DevOps in the sense that it spans the entire life-cycle and doesn't stop at deployment. In this manner, it is similar to DevOps. ITIL has a set of established practices that can help in DevOps transformation. ITSM has a set of defined processes that can be adopted for DevOps. For example, it has a detailed incident management process. This could be tailored for a DevOps organization and be used in its incident handling procedure. + +Here's some of the common ITSM processes. + +![ITSM processes](/images/blog/introduction-to-the-itsm-program-at-yale-why-are-we-doing-this-august-2011-4-638.jpg) + +The next component that acts as an enabler for the _First Way_ is CI/CD. + +### CI / CD +One of the commonly visible artifacts in the DevOps transformation is the CI/CD pipeline. But, before getting there, let's look at _what is_ CI/CD. + +#### Continuous Integration +According to [Martin Fowler's blog post](https://martinfowler.com/articles/continuousIntegration.html), +>Continuous Integration is a software development practice where members of a team integrate their work frequently, usually each person integrates at least daily - leading to multiple integrations per day. Each integration is verified by an automated build (including test) to detect integration errors as quickly as possible. + +So, CI is the practise of checking the code and ideally running it through an automated test to ensure that it hasn't broken the build. Note that the focus here is to just get the code checked into the versioning system quickly. The objective is to avoid technical debt. If a developer has to check-in code every day, handling merge conflicts is easy. If a developer checks in code after a month, resolving the merge isisues with all other changes will be extremely difficult. + + +#### Continuous Delivery +Continuous Delivery takes Continuous Integration (CI) a step further. As per another [Martin Fowler's blog](https://martinfowler.com/bliki/ContinuousDelivery.html), +>Continuous Delivery is a software development discipline where you build software in such a way that the software can be released to production at any time. + +Elaborating on this, Martin Fowler says, +>You achieve continuous delivery by continuously integrating the software done by the development team, building executables, and running automated tests on those executables to detect problems. Furthermore you push the executables into increasingly production-like environments to ensure the software will work in production. + +To achieve _Continuous Delivery_, most organizations use a __Deployment Pipeline__. + +#### Deployment Pipeline +In their book, _Continuous Deliver_, Jez Humble and David Farley, describe a [deployment pipeline](http://www.informit.com/articles/article.aspx?p=1621865&seqNum=2) in this way. +>At an abstract level, a deployment pipeline is an automated manifestation of your process for getting software from version control into the hands of your users. + +This is the place where the concept of assembly line, lean manufacturing and Toyota Production system sort of comes into manifestation. In many analogies, this piepeline is very similar to the assembly line in a manufacturing plant. The alerts that go out due to a failed build (often termed as red build) is similar to pulling the [Andon Chord](https://itrevolution.com/kata/). + +However, there are practitioners who are not convinced that IT software development is akin to assembly line. They posit that software involves innovation all through the year instead of the next year's new car model. And the model of software value stream is similar to the hub and spoke model of airlines. For more of this unique through process, please have a look at this excellent presentation from [DevOps Enerprise Summit 2017 - SFO](https://www.youtube.com/watch?v=HrEZM1Yg7Ck). + +Now that we've seen one of the _CDs_ in the form of continuous delivery, let us look at another form of _CD_. + +#### Continuous Deployment +As per [Electric-Cloud's wiki page](http://electric-cloud.com/wiki/display/releasemanagement/Continuous+Deployment#ContinuousDeployment-ContinuousDeploymentOverview) Continuous Deployment is +> the practice of continuously pushing to production new versions of software under development. + +So it takes the Continuous Delivery to a new level by actually pushing the code out to production. In both _Delivery_ and _Deployment_ code is tested in production like environments. The difference is that in _Delivery_, new change is not deployed until requested (a manual [circuit-breaker](https://www.martinfowler.com/bliki/CircuitBreaker.html)) vs _Deployment_ where the code is automatically promoted to production. + +![Continuous Delivery vs Continuous Deployment](/images/blog/differnce-1.png) + +Puppet Labs has a blog about [continuous deliver vs continuous deployment](https://puppet.com/blog/continuous-delivery-vs-continuous-deployment-what-s-diff) + +So we've seen various methods, processes and tools that can aid in a smooth flow of work from left to right. Now, let us turn the focus on the right to left feedback cycle. The end-goal of the _First Way_ is __Continuous Flow__. It means cross-functional teams can take a requirement and get it built and released as needed with no impediments. Basically it means, "deliver requirements on-demand". + +## The Second Way + +The Second way describes the principles that enable the fast and constant feedback from right to left at all stages of value stream.(_Source: DevOps handbook_) + +The purpose of the feedback loop is to ensure that we can quickly catch errors, avoid costly production issues. The Second Way is the part of DevOps that stresses on monitoring, dashboards and metrics. + +The Second way is closely tied to first way in terms of the underlying influences. e.g.: Kanban boards make work visible and SCRUM meetings help by providing a quick feedback on the impediments. By combing them, project managers can quickly feedback on project status and work on re-prioritizing, if needed. + +The Second way also requires cross-functional teams and interaction. Operations team could be part of project planning meetings and provide their (non-functional) requirements that is tracked and tested as part of the release process. This ensures a more robust product. + + +![Second Way](/images/blog/second_way.png) + +## The Third Way + +The Third Way focuses on creating a culture of continual learning and experimentation.(_Source: DevOps handbook_) + +![The Third Way](/images/blog/third_way.png) + +The Third Way is influenced by the concepts of Learning Organization, Kaizen, Improvement Kata, blameless postmortems and a lot of ideas. Let us explore a few of them. + +### Learning Organization + +The concept of a Learning Organization / Organizational Learning is quite old. It was brought into focus when the American manufacturing companies were trying to learn and compete against the Japanese organizations. One of the better definitions of a __Learning Organization__ was given by Prof. Garvin an [Harvard Business Review article](https://hbr.org/1993/07/building-a-learning-organization). + +>A learning organization is an organization skilled at creating, acquiring, and transferring knowledge, and at modifying its behavior to reflect new knowledge and insights. + +Peter Senge explained the different aspects of a learning organization using [5 aspects](http://www.thechangeforum.com/Learning_Disciplines.htm). + +![5 disciplines of learning organization](/images/blog/the-five-disciplines-senger.jpeg) + +In the context of DevOps, it means that the Value Stream of an organization is continuously adopting to the needs of the customer and changing based on the experiences, the failures and potential optimizations. + + +### Improvement Kata + +In a DevOps organization, improvement is being necessary all the time. Improvements happen in the process, tooling, requirement gathering and so on. When an organization wants to embark on the DevOps transformation, they typically use the concept of Improvement kata to target one aspect of the organization and then transform it by using the 4 step process. + +Improvement Kata is a _Lean_ tool. The idea is to improvements have to be made in a scientific and measurable manner so that the business value is optimized. +![improvement kata](/images/blog/IK%20Sticks.jpg) + + +The behavior exhibited by an organization practising these aspects is the concept of __Hypothesis Driven Developement__. In such a model, every requirement is actually considered as a hypothesis that may or may not be proven to be true. If it is not true, the team analyzes and moves on. If not, the team analyzes for the success and builds further hypothesis to make it more successful. + +![Hypothesis driven development](/images/blog/HDD_StoryCard_10a9a864fa5f5db4bbaea5bb39dddf60.jpg) + +To hear more about this concept, watch this excellend video from _DevOps Summit 2017 SFO_ about the [The Yin and Yang of Speed and Control ](https://www.youtube.com/watch?v=-Rq-fuiKNCU) + + +### Blameless Postmortem + +I guess the best way this has been described is the [Google's SRE Book](https://landing.google.com/sre/book/chapters/postmortem-culture.html) explains it. +>The cost of failure is education. + +In the context of DevOps, blameless postmortems are a form of learning organization behavior. It is both the process of knowledge management process where the root cause is explored and document and a process improvement where the underlying cause maybe fixed so that the issue does not re-occur. + + +## Beyond DevOps + +DevOps as a term was coined to primarily show that two teams that typically don't collaborate can actually work together. Once organizations have started to see the benefits, DevOps movement is now growing into partnership with other organizations as well. For example, here are a few of the common movements: + +* __DevSecOps__: The purpose and intent of _DevSecOps_ is to build on the mindset that _"everyone is responsible for security"_ with the goal of safely distributing security decisions at speed and scale to those who hold the highest level of context without sacrificing the safety required. (Source: [DevSecOps blog post](http://www.devsecops.org/blog/2015/2/15/what-is-devsecops)) +* __RuggedDevOps__: Rugged DevOps brings the lean thinking and manufacturing work of W. Edwards Deming that DevOps espouses to "ruggedizing" software. In the context of security, this means re-examining the supply chain of software you use to build your products on top of and working to build in security instead of trying to tack it on at the end in a last-ditch security review. (Source: [Want rugged DevOps? Team up your release and security engineers](https://techbeacon.com/want-rugged-devops-team-your-release-security-engineers)) +* DevOps with auditing and legal teams: This is of interest to regulated industries like banking. The idea is that the auditing functionality that can be automated is built into the pipeline. Audit trails and logging are implemented as a frame-work so that the necessary processes and artifacts for audits are baked in. + +So the idea of cross-functional teams is growing and the DevOps movement will evolve to encompass more teams as organizations see more success through a rapid innovation cycle that this movement can offer. + +This is my idea of giving back to community on the stuff I learnt about DevOps this year. Hope you find it useful! + + diff --git a/_posts/2017-2-9-csp-to-avoid-mixed-content.md b/_posts/2017-2-9-csp-to-avoid-mixed-content.md new file mode 100644 index 0000000000000..d5615e61a15e8 --- /dev/null +++ b/_posts/2017-2-9-csp-to-avoid-mixed-content.md @@ -0,0 +1,59 @@ +--- +layout: post +title: How to auto-upgrade to HTTPS (aka avoid mixed content)? +comment: true +description: When migrating to a full HTTPS site, Content-Security-Policy and Upgrade-Insecure-Requests could be your friend. This post tells you the use of the headers and the mechanism to implement it during the site migration. +image: /images/blog/csp.jpg +--- +Migrating to a full HTTPS site is hard. Using "Content-Security-Policy: upgrade-insecure-requests" can reduce the "mixed-content" errors for embedded objects. Finally, use Strict-Transport-Security header to secure the domain its sub-domains. + +![CSP logo](/images/blog/csp.jpg) + +## HTTPS Migration - The Challenge +In the recent past, there has been a lot of push to move websites to HTTPS. Google has been dangling the carrot of a better ranking by making [HTTPS as a ranking factor](https://webmasters.googleblog.com/2014/08/https-as-ranking-signal.html" target="_blank). + +However, the biggest issue is timing the migration. If the primary site moves to HTTPS and the embedded objects do not, then the browser will block the resources. It is better to move the embedded objects over to secure site and update the source code to change the reference from HTTP to HTTPs. + +Yet, source code change is a long drawn and difficult process. In such a scenario, Content-Security-Policy will be your friend. +## Content-Security-Policy (CSP) +As per [W3C, CSP](https://www.w3.org/TR/CSP3/) is: +
..a mechanism by which web developers can control the resources which a particular page can fetch or execute, as well as a number of security-relevant policy decisions.
+One of the directives is the [upgrade-insecure-requests](https://www.w3.org/TR/upgrade-insecure-requests/). When this directive is used as a header or a HTML meta-tag, the browser auto-upgrades requests to HTTPS. + +As per documents, 2 kinds of links are upgraded: + +- Passive mixed content + * Embedded links: These are the references to images, stylesheets and javascripts. + * Navigational links: These are the links placed in the tags. +- Active mixed content + * These are the AJAX calls / XHR requests + + +However, not all requests are upgraded. We learnt this the hard way during a migration. +### Gotcha# 1: Browser support +First off, not all browsers support CSP. As per [caniuse.com](http://caniuse.com/#search=upgrade), Firefox, Chrome and Opera are the browsers that support this directive. IE, Edge and Safari currently do not support it. + +![Can I use: CSP](/images/blog/can_I_use_csp.png) + +### Gotcha# 2: Exceptions +Although the W3C document mentions that navigational links are upgraded to https, both Chrome and Firefox have different interpretation + +Here's what Mozilla says [about navigation links](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/upgrade-insecure-requests): + + - Links on same domain are upgraded + - 3rd party links are not upgraded + +Chrome on the other hand [says this](https://developers.google.com/web/fundamentals/security/prevent-mixed-content/fixing-mixed-content): +
Note that having http:// in the href attribute of anchor tags () is often not a mixed content issue, with some notable exceptions discussed later.
+So Chrome will not upgrade links to HTTPS. +## Gotcha# 3: Third Parties +Third party content is not upgraded. Since browsers don't know if those domains support HTTPS, they don't upgrade. In the current versions, such content is silently blocked. You can find these blocked content by opening the developer tools in Firefox/Chrome and navigating to the console window. It would looks like this example: + +![Active mixed content errors](/images/blog/active-mixed-content-errors.png) + +## What's next? +By using the CSP header, most of the embedded object errors can be removed. CSP supports [reporting](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP#Enabling_reporting) as well. By enabling this, you as the content publisher can get the set of URLs being blocked/warned by browsers and fix in the source code. + +A subsequent change would be to use the [Strict-Transport-Security](https://www.w3.org/Security/wiki/Strict_Transport_Security) header. This header should be enabled after the migration is complete and baked in. When this is used, the browser ensures that all requests to the domain (and sub-domains) are made over HTTPS. This will eliminate the short-comings the plain upgrade header. +## How/Where to implement these changes? +As the upgrade directive and STS can be implemented with HTTP headers, you can introduce it at your web-server/proxy level or with your CDN. For more details on how CDN can help in such a setup, refer to my blog on "[How can CDN help in SEO efforts?](https://akshayrangananth.wordpress.com/2017/01/23/how-can-cdn-help-in-your-seo-efforts/)" \ No newline at end of file diff --git a/_posts/2017-3-16-mutual-auth-certificate-revocation.md b/_posts/2017-3-16-mutual-auth-certificate-revocation.md new file mode 100644 index 0000000000000..361f3c453f8fc --- /dev/null +++ b/_posts/2017-3-16-mutual-auth-certificate-revocation.md @@ -0,0 +1,44 @@ +--- +layout: post +title: Mutual auth and Certificate Revocation +comment: true +image: /images/blog/vault.png +--- + +Over the last few days, I have been working on mutual authentication / client certificates. While working on it, I learnt a few concepts around Certificate authentication and tools used to achive it. I thought it could be useful to others and wanted to share the same. + +![Vault](/images/blog/vault.png) + +## What is Mutual Authentication? +According to the [Microsoft TechNet](https://technet.microsoft.com/en-us/library/cc961730.aspx), +
Mutual Authentication is a security feature in which a client process must prove its identity to a server, and the server must prove its identity to the client, before any application traffic is sent over the client-to-server connection.
+Basically, it is a process where the client and server both have to present a certificate and both have to be verified for the TLS handshake to complete before the request/response to begin. +## Why is it different? +In the normal TLS handshake (like regular HTTPS websites), the burden of proof is on the server. That is, when I connect to my travel site, I need to be sure that I am actually talking to the travel website's server and not somebody else. I will gain my access by providing credentials. This is fine for most situations. + +However, imagine I am an agent can can do bulk bookings. I may want lines of credit and confirmed bookings before my payment goes through. In such scenario, the travel site would go through extra verifications and ask for some deposit. Finally, they may give me a client certificate to ensure that they know that is actually me doing the booking. + +As another use case, when your phone tries to talk to app store, it would need to prove that is indeed an Apple / Android device before proceeding with the app updates. The device manufacturer can embed the client certificate when the phone is manufactured so that it can be trusted. + +[CodeProject](https://www.codeproject.com/Articles/326574/An-Introduction-to-Mutual-SSL-Authentication) has an excellent article on setting up and testing this kind of a setup. +## Certificate Verification challenge +In a normal TLS handshake, the server presents it's certificate and intermediaries. Browsers have pre-loaded "Root" certificates. Using the root, the browser builds a trust chain and decides if it can trust or not trust the certificate. + +A problem with this approach is that servers may be compromised. If a TLS cerificate is revoked by a CA, the browser will never know. To work-around this, 2 techniques are in use. These techniques try to ensure that when a server sends a certificate, the browser can query the certificate authority (CA) to see if the certificate is still valid. + +- Certificate Revocation List (CRL): In this approach, the Certificate Authority (CA) publishes a list of certificates it has issued and it's status. The list is published at a fixed period or right after a certificate is revoked. The primary challenge with this approach is this CRL list keeps growing and overtime can get unwieldy. +- Online Certificate Status Protocol (OCSP): In this case, the TLS certificate will list an OCSP domain. The client can send a request to this OCSP server with the cert that it is trying to verify. The OCSP responder then says if the certificate is valid or invalid. In this case, the response is of fixed length. Request is not unwieldy as well. + +Both CRL and OCSP response can be digitally signed. OCSP provides a way to send a nonce value to reduce the risk of replay attacks. Unfortunately, not many responders support it and hence it is not very effective. +## Browser support for Certificate revocation +There is an excellent but dated article on browser support at Spiderlabs titled ["Defective by Design? - Certificate Design in modern browsers"](https://www.trustwave.com/Resources/SpiderLabs-Blog/Defective-By-Design----Certificate-Revocation-Behavior-In-Modern-Browsers/). + +According to Wikipedia and a [blog post by Maikel](https://www.maikel.pro/blog/current-state-certificate-revocation-crls-ocsp/), here's the status: + +- Safari supports OCSP checking +- IE, Opera and Firefox support CRL and OCSP. They do a soft-fail. That is, if the CRL/OCSP server is not reachable, the cert is loaded normally. +- Chrome does not perform OCSP/CRL checks directly. It can be enabled if required. Chrome believes that all methods are currently not very effective and they follow a strategy outlined in their [CRLSets](https://dev.chromium.org/Home/chromium-security/crlsets) page. + +  +## Command Line check for OCSP +Finally, if you do need to run check for an OCSP responder, Openssl has the commands to troubleshoot. [Checking OCSP revocation using OpenSSL](https://twitter.com/ivanristic?lang=en">Ivan Ristic's excellent blog post, In large-scale empirical studies that we conducted (using 500+ Alexa top mobile webpage videos collected via WebPagetest), we find that SI and PSI are linearly correlated ( at 0.91, to be precise). This means most of these webpages aren't visually jittery. In the cases where visual jitter or layout stability problems exist, SI and PSI differ quite a bit._[Perceptual SpeedIndex for measuring above-the-fold web performancee](https://www.instartlogic.com/blog/perceptual-speed-index-psi-measuring-above-fold-visual-performance-web-pages)_ + + +So the real use case would for pages where: + +* Page render is impacted due to layout changes due to injected ads +* Delayed font-loading, etc. + +Considering that page layout changes due to injected ads is a very bad practice anyway, the new metric does not help much. Another challenge would be to re-educate the performance team on a new metric just when they were beginning to learn/understand the concept of SpeedIndex. + +Although not insurmountable, the question is if it would be worth the effort of re-education when the 2 metrics correlate at over 0.9 for most use cases. + +#### Edit: 16-May-2017 +On a question around Perceptual SpeedIndex and its availability on WPT, this is what Pat Meenan says: + +>It's a variant of the Speed Index code that uses SSIM instead of histograms for checking completeness. +The main difference is that it is sensitive to page elements moving (i.e. the page shifting down a few pixels would radically change the completeness %). That could be a good or a bad thing depending on how you felt about layout moving. +Source: https://www.webpagetest.org/forums/showthread.php?tid=14883 + +## Closing thoughts +In the concluding part of the paper, the authors describe their efforts to figure out the right combination of metrics that closely approximate to the actual perceived performance. As per the study, even with a combination of all existing performance metrics coupled with SpeedIndex and Perceptual SpeedIndex, we can predict the actual perceived performance only at an accuracy of around 75%. Meaning, there is a lot of scope to figure the one metric that can exactly explain the perceived performance. + +If you are interested in a similar research that I had done, feel free to check these blogs: + +* [Metrics research - single document](https://community.akamai.com/docs/DOC-2021) +* [Metrics research - part 1](https://community.akamai.com/community/web-performance/blog/2015/05/11/what-web-performance-performance-metrics-do-i-track-part-1) +* [Metrics research - part 2](https://community.akamai.com/community/web-performance/blog/2015/05/11/what-web-performance-performance-metrics-do-i-track-part-2) +* [Metrics research - part 3](https://community.akamai.com/community/web-performance/blog/2015/05/11/what-web-performance-performance-metrics-do-i-track-part-3) +* [Metrics research - part 4](https://community.akamai.com/community/web-performance/blog/2015/05/11/what-web-performance-performance-metrics-do-i-track-part-4) + + +## Addendum +As such, these hypothesis are not ground breaking. For example, [Tammy Everts](https://www.soasta.com/blog/author/teverts/) had published a blog about the cornucopia of measurements titled *[Metrics, metrics everywhere (but where the heck do you start?)](https://www.soasta.com/blog/metrics-metrics-everywhere-but-where-the-heck-do-you-start/)* + +The argument in this presentation was that the metric to be measured varies by your role in the organization. There is never an easy answer to it and performance should cannot be boiled down to a single number. eg: Here are the metrics that are currently well known - and more keeps getting added over time! + +![Lots of metrics](https://www.soasta.com/wp-content/uploads/2015/05/metrics-grid2.png) \ No newline at end of file diff --git a/_posts/2017-4-17-Wordpress-vs-Github-Pages.md b/_posts/2017-4-17-Wordpress-vs-Github-Pages.md new file mode 100644 index 0000000000000..ec281b97e334d --- /dev/null +++ b/_posts/2017-4-17-Wordpress-vs-Github-Pages.md @@ -0,0 +1,80 @@ +--- +layout: post +title: Wordpress.com vs Github Pages Performance compared +comment: true +description: Wordpress.com gets a lot of attention about its bloat and slowness. While moving to Github pages, I ran a comparison and wanted to show the magnitude of difference across the 2 platforms. +image: +--- + +Wordpress.com gets a lot of attention about its bloat and slowness. While moving to Github pages, I ran a comparison and wanted to show the magnitude of difference across the 2 platforms. + +## Background +On the WebPagetest forums, there are a lot of questions about slowness and issues with Wordpress. For example, sample these 3 top results for my search query + + wordpress slow site:www.webpagetest.org + +* wordpress extremely slow, even on empty page +* very slow load times +* variable wordpress time to first byte +Google search result + +During last month, I moved my blog from Wordpress to Githubpages. I wrote about it in the reason and the steps in earlier post, __[Migrating Wordpress Blogs to GitHub Pages](https://akshayranganath.github.io/Migrating-Wordpress-Blogs-to-Github-Pages/)__. While I was migrating, I captured some metrics from WebPageTest to measure the magnitude of the reduction I could derive from this migration. + +For the purpose of the comparison, I used the blog post, [Useful SEO Tags - Part 1](https://akshayranganath.github.io/Useful-SEO-tags-part-1/). At the time of migration, both the posts had the exact same content and same set of embedded images within the post. + +## Blog performance compared + +Before comparing all the metrics, it is important to note that both the platforms use a CDN underneath. So the CDN can help in reducing the response times through various caching and routing optimization. For the purpose of the test, I used the following settings: + +* WebPageTest agent in Dulles - Thinkpad,VA +* User-Agent Chrome +* 9 test runs for each test + +With the test, here's the results I get. The numbers are from the [WPT comparison page](https://www.webpagetest.org/video/compare.php?tests=170325_8H_PX2%2C170325_XY_PCF&thumbSize=200&ival=100&end=full). + +| WPT Test | First byte | Start render | Visually Complete | Speed Index | Doc Complete | Fully loaded | # Req | Bytes | +| ---- | ---------- | ------------ | ----------------- | ----------- | ------------ | ------------ | ----- | ----- | +| [Github](https://www.webpagetest.org/result/170325_8H_PX2/7/details/) | 0.208s | 0.892s | 0.9s | 900 | 1.576s | 4.223 s | 44 | 569 KB | +| [Wordpress](https://www.webpagetest.org/result/170325_XY_PCF/8/details/) | 0.202s| 2.078s | 2.7s | 2205 | 3.617s | 7.876s | 84 | 1026 KB | +| _Reduction_ | _-3%_ | _57%_ | _67%_ | _59%_ | _56%_ | _46%_ | _48%_ | _45%_ | + +Across the board, I see a huge reduction. SpeedIndex reduced by almost 60% indicating above the fold content was available within a second. The dramatic improvement is better visualized in this visual progress image. +![github vs wordpress - visual progress](/images/blog/github_vs_wordpress_visual_progress.png) + +Here's a graphic on the reduction in each performance metric. +![github vs wordpress - timing metrics](/images/blog/github_vs_wordpress_timing_metrics.png) + +Finally, here's the side-by-side video of the websites loading the content. + + +## JS & CSS Complexity + +Apart from performance, the migration resulted in a huge reduction in the complexity of the JS code and CSS scripts that are used. I used the [YellowLab tools](http://yellowlab.tools/) to run the comparison. Here's link to the results: + +| Test | Score | URL | +| ---- | ----- | --- | +| Wordpress | 68/100 | [http://yellowlab.tools/result/ep126bxo5p](http://yellowlab.tools/result/ep126bxo5p) | +| GitHub | 98/100 | [http://yellowlab.tools/result/ep123gqut6](http://yellowlab.tools/result/ep123gqut6) | + +Here's some the primary reasons for Wordpress scoring so low. + +* 18 webfonts are loaded on the wordpress page +* Bad and duplicated CSS code +* 997 DOM related queries were made. This seems to be a bit outrageous for a page that is just loading blog! + +And the reason Ghithub pages lost 2 points is that Fastly CDN that hosts the GitHub page hasn't enabled HTTP/2. Once this is done, the score could be a clean 100! + +### YellowLab result for Wordpress.com site +![Wordpress.com yellow lab report](/images/blog/yellowlab_wordpress.png) + +### YellowLab result for Github pages +![Githubpages yellow lab report](/images/blog/yellowlab_githubpages.png) + +## Github Pages FTW! +Based on the numbers, Githubpages shaved off more almost half the bytes resulting in a much more peformant website. Wordpress did add a lot of scripts that were not necessary for rendering and I had no control over its use. With Github pages, I do have 2 external scripts in the form of Google Analytics and Disqus. Since I know the scripts, I can easily control. If at some point say Disqus misbehaves, I can very easily remove it when building the page and performance will improve. I had no such freedom on Wordpress blog. + +And a final thought: Despite the fact that a CDN was in play in both cases, creating a simpler and optimal page page made a huge impact to performance. So please don't consider your CDN vendor as a magician who can pull out performant website from a hat! Good performance needs to be carefully designed and implemented! + diff --git a/_posts/2017-4-18-AWSSummit-SFO-Day1-Notes.md b/_posts/2017-4-18-AWSSummit-SFO-Day1-Notes.md new file mode 100644 index 0000000000000..01a6744a74c41 --- /dev/null +++ b/_posts/2017-4-18-AWSSummit-SFO-Day1-Notes.md @@ -0,0 +1,241 @@ +--- +layout: post +title: AWS Summit - SFO - Day - 1 Notes +comment: true +--- +AWS organized the [AWS Summit at San Francisco](https://aws.amazon.com/summits/san-francisco/) on 18th and 19th of April. It was an eye-opener for me on the devOps progress and the adoption of cloud based work-flows. I had take some notes and thought of sharing it. This is the notes from Day 1 of the event. Hope it is of use to you! + +![Amazon Summit SFO logo](https://d0.awsstatic.com/events/Summits/AWS_Summit_Logo_RGB_WhiteAWS_Horiz_CityLeft_SanFrancisco.png) + +This is part 1 of the blog post. For notes from Day 2, please see the post [AWS Summit - SFO - Day - 2 Notes](https://akshayranganath.github.io/AWSSummit-SFO-Day2-Notes/). + +## The State of Serverless Computing + + +### Serverless Computing + +Built on function as a service. Building blocks are container, function as a service (FAAS) and then serverless computing + +Use cases: + +* Static and dynamic web apps +* Mobile apps +* IoT +* Big data +* Chatbots +* Media & log processing + + +>Value prop for Lambda:
** Pay only when the function runs, scaling is automated. **.
Next step evolution of using AWS. + + +>[Occupy the cloud - Distributed Computing for the 99%](https://arxiv.org/pdf/1702.04024.pdf) talks about how AWS is bringing Cloud Computing to everyone. + +* 600 concurrent functions +* Expedia 1.2 billion lambda requests per month as part of CI/CD pipeline + +#### Autodesk case study - get the PPT +very good stats on how devOps can save time and money + +* SAM - serverless CI/CD + + Code -> AWS Codecommit -> AWS Codebuild -> AWS Codebuild -> AWS Cloudformation + Code -> Commit -> Build-> Test ->Deploy to prod +![Code deploy pipeline](http://d0.awsstatic.com/product-marketing/CodeBuild/CodeBuildCodePipeline2.png) + +AWS Cloud X-Ray: debugging tool + +---- + +## Building Serverless Web Applications + @leozh - Solutions Architect + +Basic Lambda concepts + +* no servers to provision or manage - C#, NodeJs, Python, Java +* scales with usage +* never pay for idle +* availability and fault tolerance built in + +Core concepts: + +* Event driven +* Continuous scaling +* pay by usage + +*Use case*: Create a thumbnail of a just updated image. Github example here: [https://github.com/awslabs/serverless-image-resizing](https://github.com/awslabs/serverless-image-resizing). + +New features: + +- Allows for creating dev/prod - by pulling in the latest +- Key-value pairs (SAM) + +#### Amazon API Gateway + +* authenticate and authorize + * sigv4, oAuth, Amazon Cognito, custom auth ==> per method authorization +* unify multiple microservices +* DDoS protection / Scaling: Use cloudfront on top if API gateway, tie WAF rules, use Shield to protect origin +* Stage variables have environment variables (like prod, dev). based on it, point to beta or prod end point. + +#### Design patterns + +* _Monolithic_: don't use! + * One large lambda function. Easier to comprehend. + * Single handler - so single one for GET/PUT/POST, etc. +* _Microservices_: provides a lot of freedom to make changes and impact small sections. + * Each method has one lambda function. one each for GET, PUT, POST + * Easier to code and focus on one smaller component. Focus shifts to integration testing. + * Use X-ray for debug, Use SAM to manage Lambda functions + +<< get image from PPT >> + +SAM is an extension of CloudFormation. _CloudFormation_ is basically __infrastructure as code__. Supports JSON or YAML. + +* Claudia.js - nodejs for this purpose +* Chalice - python + +Code example at [Github](https://github.com/awslabs). + +Amazon has also built a pipeline automation for CI/CD. Here's the refarch: [https://github.com/awslabs/ecs-refarch-continuous-deployment](https://github.com/awslabs/ecs-refarch-continuous-deployment). + +------ + +## Serverless Orghestration with AWS Step functions +Ability to connect multiple functions into a work-flow using GUI. Essentially, chaining Lambda or any kind of function to follow a strict logical ordering. + +> To learn about how to build app in the modern world, read this: [https://12factor.net/](https://12factor.net/). + +Coordination must have + +* scales out +* doesn't lose state +* deals with errors/timeouts +* easy to build out. + +** AWS Step function - State machine as a service ** + +* Define in JSON +* Visualize in console +* monitor executions + +#### Sample state machine +![State machine](https://github.com/awslabs/lambda-refarch-imagerecognition/raw/master/images/step-function-execution.png) + +7 states for lambda step function + Task, choice, parallel, wait, fail, succeed, pass + +Once the state is executing, you can go back and see the execution in UI. It will show the specific inputs, the states reached and final output. + + DevOps Best Practice + If you build a JSON spec, then: + 1. publish the spec + 2. provide examples + 3. create a LINT and publish that as well. + +---- + +## Building CI/CD Pipelines for Serverless Applications + +Serverless computing space: [https://aws.amazon.com/serverless/](https://aws.amazon.com/serverless/) + +Typical work-flow: + + Event Source -> Function (lambda) -> Services (anything) + +Lambda supported on + +- Node.js +- Python +- Java +- C# + +e.g.: deploy code through slack or even Alexa + +* _Cont Integration_ - anytime code is in main-line, it will built and it is valid +* _Cont Delivery_ - automatically build, test and after a manual approval, push to prod. +* _Cont Deployment_ - automatically build, test and push to prod - no manual gating. + +**Amazon Cloudformation** is a good way to implement CI/CD. However, it can get verbose. An alternative is to use __Serverless Application Model (SAM)__. + +* Have different accounts for Dev, Stage, and Prod +* Having a single account with different stacks is ok for smaller team. +For more details, refer [AWS Organizations](https://aws.amazon.com/organizations/). Basically, control access to AWS environments by policy rather than trying to handle everything manually. + +#### AWS Codebuild +Can be used as a managed build service. Pay by minute. Can provide docker images for build. +*cheaper than using an EC2 based jenkins or build pipeline since it is charged by hour.* + +#### Localstack +Altassian has developed a [local EC2](https://github.com/atlassian/localstack) stack. Use it to test locally before moving to EC2. +*This is cool!* + +---- + +## Deep Dive on CICD and Docker +- John Pignata - Startup solutions architect + +Slides: https://www.slideshare.net/AmazonWebServices/srv412-deep-dive-on-cicd-and-docker-75232162 + +#### CI/CD Objective + How can we quickly and reliably deliver good ideas to our customers? + +<< Learnings slide, CI/CD slide - get it when published. >> + +ECR - docker repository in CI/CD scenario. + +#### AWS CodeBuild +Build and test projects. How does the testing work? 4 phases: + +- _install:_ get the right test libraries, etc +- _pre-build:_ log into the right AWS ECR etc +- _build:_ run the actual build process, SASS +- _post_build:_ build the final jar, clear up temp files, push to registry + +##### Tip! + + Tag the code image with something that links back to development. Avoid using simplistic names like "LATEST", "PROD". + +#### Amazon EC2 Container Service (Amazon ECS) +Managed service for setting up docker images. + +- Able to specify the min health required, maximum health. ECS will straddle the deployment based on these numbers. +- Can even perform canary deploy +- Blue/Green deployment: Having 2 different app versions (blue / green). Basically launch for short time, measure and stabilize or revert back. + - Common way is with DNS swap. + - New feature: Host based routing. + +#### AWS CodePipeline +Provides ability to model and implement the full CI/CD pipeline. It can talk to external tools and can even integrate with Lambda functions. + + +Reference architecture: [https://github.com/awslabs/ecs-refarch-continuous-deployment](https://github.com/awslabs/ecs-refarch-continuous-deployment) + +![CodePipeline](https://github.com/awslabs/ecs-refarch-continuous-deployment/raw/master/images/architecture.png) +----- + +## Deep Dive: Log analytics with Amazon Elasticsearch Service + +* Use [ElasticSearch](https://www.elastic.co/products/elasticsearch) for digesting log. +* Use [Kibana](https://www.elastic.co/products/kibana) for visualizing data. + +Amazon Elasticsearch is a managed service around Elastic Search. +{Used by Expedia} + +Use "Data Pattern" slide for sizing of the EL cluster. + +__Usage__ + data source -> Amazon Kinesis Firehose -> Amazon Elasticsearch service -> Kibana + +> Take the best practice slide + +#### Kinesis Firehose +send streaming data between applications. + +- serverless +- error handling +- s3 backup + + +Sign the requests on the fly using a local proxy + aws-es-kibana diff --git a/_posts/2017-4-18-AWSSummit-SFO-Day2-Keynote-Notes.md b/_posts/2017-4-18-AWSSummit-SFO-Day2-Keynote-Notes.md new file mode 100644 index 0000000000000..5757c137dc79b --- /dev/null +++ b/_posts/2017-4-18-AWSSummit-SFO-Day2-Keynote-Notes.md @@ -0,0 +1,128 @@ +--- +layout: post +title: AWS Summit - SFO - Day-2 Notes from Keynote and Fireside chat +comment: true +--- +AWS organized the [AWS Summit at San Francisco](https://aws.amazon.com/summits/san-francisco/) on 18th and 19th of April. It was an eye-opener for me on the devOps progress and the adoption of cloud based work-flows. I had take some notes and thought of sharing it. This is the notes from Day 2 of the event. Hope it is of use to you! + +![Amazon Summit SFO logo](https://d0.awsstatic.com/events/Summits/AWS_Summit_Logo_RGB_WhiteAWS_Horiz_CityLeft_SanFrancisco.png) + +This part 2 of the blog post. For notes on the 1st day's events, see the [AWS Summit - SFO - Day - 1 Notes](https://akshayranganath.github.io/AWSSummit-SFO-Day1-Notes/) post. + +# Keynote featuring Dr. Werner Vogels, CTO, Amazon.com + +- 7,500 people in attendence +- $ 14 billion run-rate growing at 14/40% YOY? +- focus on startups +- Twilio literally used every service of AWS. +- new "in the cloud" integrators instead of System Integrators like Cognizant, IBM, Accenture, etc. +- Software vendors - almost everyone is on AWS. 1,200 ISVs +- SaaS integrations - Splunk is a strategic partner +- Yelp - uses Yelp for end to end delivery on timing of the delivery of content. +- Developers are like James Bond and AWS is like the _'Q'_ + - *Supersonic speed*: over 90 services. Mostly on feedback. + - AWS Codestar: In one environment, IDE, + - Create a project, add the roles, build the piepline and connect with IDE. + - F1 instances: Field programmable gate array. Real time video processing, financial analytics, big data search and analytics. + - End goal - make the infrastructure invisible. + - Managing containers is hard - almost like pre-cloud world. + - Instead use ECS. (Read blog [Mapbox moves to ECS and you won't believe what happened.](https://www.mapbox.com/blog/switch-to-ecs/)) + - NextDoor - private social network. + - Entire platform is on AWS. + - build and deployment time down by 10% + - used to have weekly releases + - Red+Black deployments: Every deployment, create a whole set of new EC2 instances. Route a small traffic and based on feedback, switch the traffic and bring down old instances. + - dropped deployments from 25 mins to 7 mins. + - 10x improvement in deployment. dozens of deployments a day! + - Serverless - Lambda + - Sholastic, Robot, Netflix, Experian - some examples + - AWS X-Ray: UI based debugging and troubleshooting on flow, components. + - Amazon DynamoDB: auto scales: Expedia, Lyft + - Amazon DynamoDB Accelerator (DAX): fully managed, in-memory cache + - *Flight* + - Old world database: Expensive, proprietary, lock-in + - OpenSource Dbs to help. + - 23,0000 DB migrated from on-prem to cloud + - Amazon aurora: MySQL compatible. 5x performance. Amazon Aurora PostGreSQL: semantically closer to Oracle. + - Aurora - used by Expedia. 100 million writes a day, 17,000 writes a second. 17ms read. + - *X-Ray Vision* + - Amazon Athena - interactive query service. Amazon EMR - hadoop, spark, presto. Date warehousing - RedShift. + - Yelp, Hudl, Sholastic, sling. + - Redshift Spectrum: Directly run data warehouse queries on S3 data. Yelp is using it. + - *Precognition* + - Amazon ML (machine learning), + - Deep learning AMI + - Amazon Rekognition used for C-SPAN politician recognition and tagging. 3 weeks saving 9,000 hours! + - Image Rekogniition - scoring for inappropriate content. + - Amazon Polly - 47 voices, 24 languages, customize on what is said., whispering voice and speech marks. + - Amazon Speech recognition - Amazon lex. Lex connected to Lambda. So based on the intent of the voice, this will be used to trigger the Lambda function. + - Slack uses Cloudfront, Athena, + - Amazon lex has been integrated with slack. Conversational bots. + - Amazon AI - hubspot, duolingo + - *Immortality* + - { lot of focus on startups } + - moving fast, agile, use all of the services + - Digital Transformation: ticketmaster, PBS, GE + +# Fireside Chat featuring Andy Jassy, CEO, Amazon Web Services – RED + +- growth of 47% +- Workday - preferred cloud provider. +- private cloud - not common in lot of customers. Hybrid operation is more common. +- [VmWare + AWS collaboration](http://www.businesswire.com/news/home/20161013006574/en/VMware-AWS-Announce-Hybrid-Cloud-Service-%E2%80%9CVMware) +- Multi cloud strategy: Most customers don't do it. + - Standardize to lowest common stuff - that's constraining their developers + - Learning different platforms ==> Isn't this is the same as vendor lock in? + - Diminish buying power - essentially lose the volume discounts. +- Machine learning / deep learning - 3 levels + - Level 0: actual mathematicians / expert practitioner (eg: Netflix) + - Level 1: Providing tools for regular developers. + - Level 2: Quick tools for everyone. +- Big challenge is getting data from data center into the S3. +- __Connected devices__: Over time servers at offices or homes will be replaced with connected devices. That will be the new hybrid model. + - Tata motors manages trucks ? +- *Amazon GreenGrass* - software module that can run lambda functions that will partly run in cloud and in the device. +- Focus + - expansion of geographical foot print: in every tier 1 country + - DB expansion - Aurora fastest growing service + - Container and orchestration management + - Event driven and serverless computing + - Self service and improving it. +- How do you innovate so quickly? + - disproportionately hire builders + - understand that launch is a starting line and not the finish + - organization + - small autonomous team + - technology and product management in the same team + - teams own the road-map + - eat your dog food + - amazon.com itself is using AWS and that gives a large amt of feedback + - Trying to get away from the institutional ways to say "no". + - About ability to make fast decisions. +- Organizational culture + - unusually customer oriented + - most tech companies are competitor focused. + - we are pioneers + - most competitors are losing their drive to innovate + - unusually long term oriented + - trusted advisor capabilities in the customer relationship team + - we'll reach out to tell customers to tell you're not using everything. So save the money. Saved $350 mil last 2 yrs to customers. +- Quirky parts of culture + - ban powerpoints at meetings + - have conversation. 6 page narrative. + - read this at a meeting and then start the discussion. + - a press release, FAQ and documentation is first created before a product is even written. + - forces to think on use and reason for its adoption. + - unusually truth seeking + - disagree and commit +- Learning + - don't fight gravity. If you know that the inevitable will happen, then you should cannibalize before your competitor does it. + - explained within the context of whether Amazon should support 3rd party suppliers and Amazon's jump to support them before eBay and others could kill them. + +# Deep Dive on AWS IoT + +- MQTT is the protocol used, WebSockets is what the phones use. +- Use a shadow device for sending the control commands. This will ensure that the device gets the commands in right order. +- [Amazon Cognito](https://aws.amazon.com/cognito/) for authentication of devices. + + diff --git a/_posts/2017-4-4-5-Free-Tools-For-DIY-SEO-Analysis.md b/_posts/2017-4-4-5-Free-Tools-For-DIY-SEO-Analysis.md new file mode 100644 index 0000000000000..c534808eba3a8 --- /dev/null +++ b/_posts/2017-4-4-5-Free-Tools-For-DIY-SEO-Analysis.md @@ -0,0 +1,85 @@ +--- +layout: post +title: 5 tools for DIY SEO Health Check +comment: true +description: Basic SEO analysis for technical folks can sound difficult. There are a lot of free tools that can help. I show 5 such SEO tools/methods for DIY SEO analysis. +image: /images/blog/health-check.jpg +--- + +SEO Analysis can appear to be quite difficult, especially if you are from a technical background and don't have a lot of knowledge of marketing techniques. However, there are many aspects of SEO that ultimately boils down to a technical health check of your website. In this blog post, I want to cover 5 free tools & methods to get you started with some basic SEO health check. + +![health check](/images/blog/health-check.jpg) + +## 1. Search Engine Check +As a first step, the first thing to check is find out if your site is even being indexed by search engines and if so, which are the pages it considers as most relevant. On google, this involves using the __*site:*__ keyword. + +For Google search, I also like to de-personalize the query to get more general result. [Moz has a blog)[https://moz.com/blog/face-off-4-ways-to-de-personalize-google] on different methods to do this. I just use the query string method since it is simple and gets the work done. To use the method, simply append the parameter __*pws=0*__ at the end of the URL after initiating the search. + +Here's the final search query I have for different search engines: + +* [https://www.google.com/#q=site:akshayranganath.github.io&pws=0](https://www.google.com/#q=site:akshayranganath.github.io&pws=0) +* [https://duckduckgo.com/?q=site%3Aakshayranganath.github.io&t=ffsb&ia=web](https://duckduckgo.com/?q=site%3Aakshayranganath.github.io&t=ffsb&ia=web) + +### Why use it? +When you perform the search, look at the search results. Sometimes, pages that may not be so important as per your business goals may inadvertantly rank high. It would be a good idea to re-examine the reason for such a ranking and fix the link structure so that the right pages are ranked. + +## 2. WooRank SEO recommendations +![Woorank logo](https://s3.amazonaws.com/woocms.woorank.com/2016/Nov/woorank-1478176499839.png) + +[WooRank](https://www.woorank.com/) provides a free report for SEO analysis. This report can't be easily exported but, it is till worth a check. It gives you details around: + +* Use of meta tags and current recommended length +* Use of title tag and its effectiveness +* Keyword cloud based on the page being analyzed +* Errors if any +* Social media interactions +* Presence of robots.txt and sitemap.xml +* Cleanliness of URLs (overuse of query parameters is considered unclean) +* Mobile compatibility + +### Why use it? +This is a very good starting point for anlyzing the home page. This report can give you questions to ask and enough data to start building a baseline of your website and the context of your webistes traffic sources and keywords. + +## 3. Screaming Frog +![Screaming frog logo](https://media.licdn.com/media/p/2/000/1ac/3dc/17d7797.png) + +[Screaming Frog](https://www.screamingfrog.co.uk/seo-spider/) is an SEO spidering tool. It can look through the website and report on a number of actors. The free version of the tool can spider 500 URLs and report on a number of factors: + +* Page Title +* Meta-description +* List of <h1>, <h2> and <h3> tags +* HTTP response codes, hreflang and more + +### Why use it? +The tool is very useful to find pages that don't have good SEO tags like <title>, <meta-description< and misconfigured redirects. I use this as a good starting point to quickly look at a site setup and how easily it is setup to be crawled. If there is an important page and it is not discovered within the first 500 URLs then maybe it is worth checking the site setup. + +## 4. SimilarWeb +![Similarweb logo](https://qph.ec.quoracdn.net/main-qimg-aa478aa0067d12b2c10c968151350ce1) + +SimilarWeb gives a very neat report on the traffic details especially around the different sources of traffic. I like the **Traffic Sources** report. It an give a pointer on where people are coming from and more importantly where they are not. This can lead to some good questions on assumptions being made about ideal user base vs the real users coming to the website. + +The comparison of similar sites too gives an idea on what's being considered as competition and it could also be a good learning to emulate the best competitors. + +### Why use it? +Use this tool for: + +* Identifying traffic sources +* Referral traffic - this can be a very good avenue to create more back links. +* Search report for top performing keywords + +## 5. Moz Free tools +![Moz logo](https://dc8hdnsmzapvm.cloudfront.net/assets/images/about/brand/moz-white-thumbnail.jpg) +Moz is the go-to resource for my learning materials on SEO. They also have some of the most robust tools for planning campaigns, building the right keywords and a lot more. However, many of the tools are paid and generally require a good understanding of marketing campaigns, content creation and other core aspects of SEO. + +However, Moz does offer a set of [free tools](https://moz.com/free-seo-tools). I like the [Moz Keyword Explorer](https://moz.com/explorer) and [Open Site Explorer](https://moz.com/researchtools/ose/). + +### Why use it? + +* __Keyword Explorer__: This tool is useful when creating content. It can give some very nice ideas on the right kind of titles and keywords to target. It can give you information on keyword complexity, the opportunity for the keyword and potential traffic for the it. +* __Open Site Explorer__: This tool provides details from the indexed data about a website. It can provide data that is similar to SimilarWeb. The only issue I have seen is that this tool has data for more popular sites. Small blogs may never be scanned by this tool. + +## Finally.. +In the end, it comes down to your use case. Depending on what you are trying to accomplish, a combination of these tools can help you get started on the path to SEO. When combined with the power of Analytics, you will be well on your way to derive some good corelations. Once you have them, you can start to test and improve the website traffic! + +In an upcoming part, I'll explore the use of Analytics packages from an SEO perspective. + diff --git a/_posts/2017-5-2-Synthetic-Testing-With-WPT-and-Jenkins.md b/_posts/2017-5-2-Synthetic-Testing-With-WPT-and-Jenkins.md new file mode 100644 index 0000000000000..d2ac7cd0cf10b --- /dev/null +++ b/_posts/2017-5-2-Synthetic-Testing-With-WPT-and-Jenkins.md @@ -0,0 +1,234 @@ +--- +layout: post +title: Synthetic Testing and with WPT and Jenkins +comment: true +description: Webpagetest is very good for monitoring website performance. By using Jenkins to run synthetic tests on WPT, it is possible to build a decent solution for automated performance monitoring and validating your performance budget. +--- +Webpagetest is an indispensable tool for performance monitoring of websites. Using the scripted test+REST API and combining with Jenkins, WPT can be converted to a synthetic testing tool to test scenario beyond the scope of tools like Gomez and Catchpoint. + +Mad scientist + +While working on a issue, I faced an situation where we had to develop a testing solution over WPT. Gomez could only run tests from a HAR file. Catchpoint could execute a transaction but was limited to Chrome. Since we had no other tool that could cover all the devices and use cases, we scripted the solution. + +I had written a very closely related article on using [Jenkins for running performance tests](https://akshayranganath.github.io/Performance-Testing-with-Jenkins/). This blog is more on creating synthetic testing with Jenkins and WPT. + +## Background +We had to measure the performance of a particular website by going through a transaction. The challenge was to ensure that the test was run on at least these 4 devices: + +- Chrome +- Firefox +- iOS (iPhone) +- an android device + +We needed the test to run for over 3 days at a regular interval of half an hour in between the tests. The gap was necessary so that the transaction would not be marked suspicious. Running the test in a recurring fashion was easy to solve. We could either use a cronjob or work with [Jenkins](https://jenkins.io/). In this instance, I picked Jenkins simply because I wanted to learn and it stores a log of the different run that I can leverage. + +So here goes the steps I followed to build a synthetic testing solution. + +## Build Synthetic Tester + +### Step 1: Create scripted WPT Test +WebPageTest offers an ability to run scripted tests. Using scripting, it is possible to design complex work flows like login, shopping cart check-out process or any kind of a transaction. Each step is then measured separately or they can all be combined for a holistic waterfall. For this blog, let's say the steps are: + +1. Go to this blog's [home page](akshayranganath.github.io) +2. Click on one of the blog links to land on the blog post page + +WPT script for the transaction would as follows. Here's a [sample report](https://www.webpagetest.org/result/170502_V8_2db53ef7be2df0371e59c5b7c7a9e597/). +``` +navigate https://akshayranganath.github.io/ +execAndWait document.getElementsByTagName('a')[4].click(); +``` + +The waterfall is broken into 2 steps. +![wpt waterfall](/images/blog/wpt_combined_steps.png) + + +### Step 2: Convert scripted test to ReST API +WPT tests can be run through [REST API](https://sites.google.com/a/webpagetest.org/docs/advanced-features/webpagetest-restful-apis). To run the API, you will require an API key. After signing up on the __[Request API Key](http://www.webpagetest.org/getkey.php)__, you should receive an API key. I had such an API key. Using this, I built a REST API call to run the test. + +In this python script, I make 3 calls. I have added a gap of 30s just to demonstrate that it is possible to delay the test runs. This is a bit crude since WPT itself will queue the tests. However, adding the 30s delay will ensure that my tests are _at least_ 30s apart. + +```python +import requests +import datetime +import time + +def runTest(location, http2=False): + key = '' + post_data = "script=navigate%09https%3A%2F%2Fakshayranganath.github.io%2F%0AexecAndWait%09document.getElementsByTagName(%27a%27)%5B4%5D.click()%3B&private=1&f=json&fvonly=1&video=1&medianMetric=SpeedIndex&k=<>&location="+location + if http2 == True: + post_data += '&cmdline=--disable-http2' + + wpt_url = "https://www.webpagetest.org/runtest.php" + + + headers = { + 'Content-Lenght': str(len(post_data)), + 'Content-Type': 'application/x-www-form-urlencoded' + } + + resp = requests.post(wpt_url, post_data, headers=headers) + + if resp.status_code == 200 and resp.headers['Content-Type'].find('application/json')>-1: + print datetime.datetime.now(), location, + if 'data' in resp.json(): + print resp.json()['data']['testId'], + print '' + +if __name__ == "__main__" : + runTest("Dulles") + time.sleep(30) + runTest("Dulles:Firefox") + #run without H2 enabled + runTest("Dulles",True) +``` + +### Step 3 Scheduling the test run +Once the script was ready, I scheduled it with Jenkins. +![Jenkins logo](https://wiki.jenkins-ci.org/download/attachments/2916393/logo-title.png?version=1&modificationDate=1302753947000&api=v2) + +For this purpose, I created a new build. In the build information, I had these 2 fields: + +- *Build*: + +```bash +python +``` + +- *Build periodically*: To ensure the script ran every 30 mins, I used the string. + +```bash +H/30 * * * * +``` + +Each time the test ran, it produced an output of this format: + +```bash +2017-04-25 02:14:00.670719 Dulles 170425_N4_8322f5c3e0261a24adbaa43d2058db6f +2017-04-25 02:14:30.906648 Dulles:Firefox 170425_XF_c0ead0e4252518366ebe4c783a2c039c +2017-04-25 02:14:31.080146 Dulles 170425_JJ_aed5b933e3238907c7835bab79c4721d +Finished: SUCCESS +``` + +That's it. The synthetic testing tool was now ready. + + +## Extracting result +WebpageTest results are available through the URL format https://www.webpagetest.org/jsonResult.php?test=. For example, here's one JSON result URL: [https://www.webpagetest.org/jsonResult.php?test=170502_V8_2db53ef7be2df0371e59c5b7c7a9e597](https://www.webpagetest.org/jsonResult.php?test=170502_V8_2db53ef7be2df0371e59c5b7c7a9e597). + +Since the results are accessible with no special authentication, I had to write a script to pull the following from Jenkins: + +* Fetch the start and end run number. Basically, each number will be one instance of 3 WPT tests being run. +* From the console output of each run, extract the WPT result ID. +* Go to WPT and fetch the result for the test run +* Extract the relevant fields and print it. In my case, I print it in a CSV friendly format. + +On Jenkins, the console output for each build has the following URL format + + http(s)://your-jenkins-domain/job//build-number/console + +Using the pattern, it was possible to script a code that would pull all the results and then dump in a CSV format. Here's the code I used to achieve this. A lot of code is hard coded but, this is my first attempt :-) + +```python +import json +import requests +from bs4 import BeautifulSoup +import time +import sys +import traceback + +def print_result(data=None, url=None, protocol='h1'): + try: + if data==None: + with open('result.json') as result_file: + data = json.loads(result_file.read()) + + browser = data['data']['from'] + nflx_home_page = data['data']['runs']['1']['firstView']['steps'][0] + #nflx_home_page = data['data']['runs']['1']['firstView'] + + + #print json.dumps(nflx_home_page,indent=5) + + metrics = ['base_page_ttfb','firstPaint','render','visualComplete','domInteractive','loadEventStart','fullyLoaded','docTime','SpeedIndex','bytesOut','requestsFull','image_total'] + print url, + print "\t "+ browser + "\t " + protocol +'\t', + for metric in metrics: + print str(nflx_home_page[metric])+'\t', + print + sys.stdout.flush() + #now try to print the timing information + except TypeError: + print 'Unable to parse JSON' + pass + except KeyError, e: + traceback.print_exc() + #print json.dumps(nflx_home_page, indent=4) + pass + except IndexError, e: + print e + #print json.dumps(nflx_home_page) + pass + except UnboundLocalError, e: + print e + pass + +def get_result(data = None): + if data == None: + with open('op.json') as result_file: + data = json.loads(result_file.read()) + #print url + try: + url = data['data']['jsonUrl'] + resp = requests.get(url) + if resp.status_code==200: + print_result(resp.json()) + except requests.exceptions.ConnectionError: + print 'Fetching failed for url: ' + url + print 'Unable to connect to network. Please check if you are online.' + except TypeError: + print 'Unable to parse the results' + print data.json() + + +def fetch_result_urls(jenkins_url, start_event=1, end_event=10, wpt_base_url="https://www.webpagetest.org/jsonResult.php?test="): + for testId in range(start_event,end_event): + url = jenkins_url + str(testId) + '/console' + resp = requests.get(url) + if resp.status_code == 200: + #fetch the result and then parse the console output + html = BeautifulSoup(resp.text,"lxml") + lines = html.findAll('pre')[0].getText().strip().split('\n') + result_no = 0 + for line in lines[len(lines)-4:len(lines)-1]: + wpt_result_status_url= wpt_base_url + line.strip().split(' ')[-1] + resp = requests.get(wpt_result_status_url) + f = open('resp.json','w') + f.write(json.dumps(resp.json())) + f.close() + if result_no < 2: + print_result(resp.json(),wpt_result_status_url) + else: + print_result(resp.json(),wpt_result_status_url,'h2') + result_no += 1 + #break + #sleep for 2 s + time.sleep(2) + +if __name__ == "__main__": + #get_result() + # get the results for test runs 1 through 50 + fetch_result_urls('http(s)://your-jenkins-domain/job//',1,50) + #print_result() +``` + +The output would be a comma-separated value of these fields: + WPT Browser Protocol TTFB FirstPaint Render VisualCompelte domInteractive loadEventStart fullyLoaded docTime SpeedIndex bytesOut requestFull image_total + +## Conclusion +By running this scripted test, it is possible to build a synthetic testing solution that can be more complex than offered by some commercial solutions. It is also free and can help get the job done when there is a rush to address some specific performance issue. + +Even when good synthetic tests are setup, this will still help. If a particular test is slow, it will be easier to go into the corresponding WPT test and look at the resources and see if something had gone wrong. For CDN specific testing, it is possible to add special headers to get back caching status. + +![Using WPT book cover](https://www.safaribooksonline.com/library/cover/9781491902783/) + +As everyone keeps saying, with WPT, the possibilities are endless! So this is just a peek at what can be done. If you are not familiar with WPT, do consider the book _[Using WebPageTest](https://www.safaribooksonline.com/library/view/using-webpagetest/9781491902783/). It is a wonderful resource to learn about the multi-faceted functionality offered by this great tool! diff --git a/_posts/2017-5-25-Markdown-for-notes.md b/_posts/2017-5-25-Markdown-for-notes.md new file mode 100644 index 0000000000000..7c6a4db921c14 --- /dev/null +++ b/_posts/2017-5-25-Markdown-for-notes.md @@ -0,0 +1,43 @@ +--- +layout: post +title: Better notes with Markdown and SublimeText! +comment: true +description: Learn why Markdown is a good for taking notes during trainings and meetings and plugins for Sublime Text2+Markdown integration. It's very good for converting to blog posts with Github pages as well. +image: /images/blog/markdown.png +--- + +Over the past few months, I have realized that Markdown is a very good way to take notes, especially if the end goal is to quickly format and convert it to a blog post as well. +![i love markdown](/images/blog/markdown.png) + +## What is Markdown? +[Markdown](https://daringfireball.net/projects/markdown/) is a syntax that was introduced by John Gruber as an easy way to convert plain-text to (X)HTML. Among other this is the default format used for publishing [GitHub pages](https://pages.github.com/). So all the blog posts on this website is written using Markdown. + +As I started to use this format and got a hang of it, I realized that it is a very useful format for taking notes as well. It is this habit that lead me to easily publish my blog posts on the AWS Summit [here](https://akshayranganath.github.io/AWSSummit-SFO-Day1-Notes/) and [here](https://akshayranganath.github.io/AWSSummit-SFO-Day2-Keynote-Notes/). + +## Markdown for taking notes +When attending meetings or training, we invariably take notes. In most cases, it is as some plain-text file or a word document. Typically, I don't like Word document since there is time spent on getting the right formatting. Plain-text is a bit boring to read since it is so plain-in-text! + +Enter Markdown! I can now take notes in simple plain text and just add the formatting within in. In most cases, I need to use the following formating tools: + +* Headings +* Images +* Links +* Occasionally - tables +* Code formatting and unformatted text + +Once you use Markdown for a few months, the syntax just becomes a second habit. While learning, one of the best resource I found was Adam Pritchard's [Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet). + +## Previewing Markdown page +Markdown doesn't need a lot of resources to be converted to HTML. There are [CLI tools](https://github.com/showdownjs/showdown), [online editors](http://dillinger.io/) and more. In my case, I use Sublime Text. So the logical option was to find the best way to get this done. + +I followed the steps outlined in the article [How to Set Up Sublime Text for Markdown Editing](http://plaintext-productivity.net/2-04-how-to-set-up-sublime-text-for-markdown-editing.html). At the end of it, I had installed a package named "Markdown Extended" and added a key-binding to show Markdown pages as a HTML on my browser. Here's my key binding file after the changes + +```javascript +[ +{ "keys": ["alt+m"], "command": "markdown_preview", "args": {"target": "browser", "parser":"markdown"} } +] +``` + +Once I had this done, there was no stopping back! Every note taken at every meeting became a markdown file that could be checked in to repository, or converted to a blog post and start a conversation! + +Hope you find this useful too. \ No newline at end of file diff --git a/_posts/2017-5-30-How-to-create-dev-environment-for-python.md b/_posts/2017-5-30-How-to-create-dev-environment-for-python.md new file mode 100644 index 0000000000000..30e1b6f94be98 --- /dev/null +++ b/_posts/2017-5-30-How-to-create-dev-environment-for-python.md @@ -0,0 +1,121 @@ +--- +layout: post +title: DevOps : Building a Python virtual environment +comment: true +description: When embarking on a devOps journey, getting a consistent development environment is the key. In this post, I show how to setup a virtual environment and install the correct libraries necessary for a project. +image: /images/blog/nut-2051596_640.jpg +--- + +When embarking on a devOps journey, getting a consistent development environment is the key. In this post, I show how to setup a virtual environment and install the correct libraries necessary for a project. + +![nut ball and bearing](/images/blog/nut-2051596_640.jpg) + +## Virtual Environments: VirtualEnv + +>A Virtual Environment is a tool to keep the dependencies required by different projects in separate places, by creating virtual Python environments for them.
+Source: https://python-guide-pt-br.readthedocs.io/en/latest/dev/virtualenvs/ + +The tool in python is called __virtualenv__. + +To work with isolated environments, you'd need to follow these steps: + +- **Initial Setup** + - Install the virtualenv tool. + - Initialize your environment + - Install dependencies + - Create the **requirements.txt** to remember the dependencies + - Check-in code and the requirements.txt file to code repository +- **Developer installs** + - Get the latest code + - Install virtual environment + - Install dependencies + - Start working on code + + +Let's look into the steps in detail. + +## Initial Setup + +Before commencing the project, one of the developers can follow these steps to help get the team going. The first step is to install the *virtualenv* tool. + + pip install virtualenv + +The next step is to initialize virtual environment on the project folder. + + virtualenv myproject + +This step will install python, pip and basic libraries within the project folder, _myproject_. After version 1.7 of virtualenv, it will use the option __--no-site-packages__ by default. It means that virtualenv will NOT install the libraries available globally. This will help ensure that the project package will contain the bare minimum libraries and the libraries installed for the specific project. + +After this, you will need to activate the virtual environment to start working on the project. + + source myproject/venv/bin/activate + +Once you do this, your prompt will change to show that you are now working in a virtual environment. It should look something like this. + + (venv) user@machine $ + +Now, go ahead and install the libraries that may be required for the project. If all the dependencies can be named at this stage, it will simply make it easier to replicate the environment. The dependency list can always be updated as I'll show in later. + + pip install + +Once done with the installations, create a list of the libraries and their versions. + + pip freeze > myproject/requirements.txt + +This file *requirements.txt* will hold the necessary dependencies for your project. In one of my projects, it looks like this: + +``` +appdirs==1.4.3 +asn1crypto==0.22.0 +cffi==1.10.0 +cryptography==1.8.1 +enum34==1.1.6 +idna==2.5 +ipaddress==1.0.18 +jsontree==0.4.3 +ndg-httpsclient==0.4.2 +packaging==16.8 +pyasn1==0.2.3 +pycparser==2.17 +pyOpenSSL==17.0.0 +pyparsing==2.2.0 +requests==2.14.2 +six==1.10.0 +urllib3==1.21.1 +``` + +This fill will need to be checked in to your project code. At any time, if any other developer installs a new library, it will have to be added to this requirements.txt file. That way once other developers pull the update from the source code repository, they will be made aware of the new dependencies. + +![cog wheels](/images/blog/cog-wheels-2125169_640.jpg) + +## Developer Install + +For each developer, the project setup is now straight-forward. They will need to install virtualenv. + + pip install virtualenv + +Then create a folder and activate virtual environment. + + virtualenv myproject + source myproject/venv/bin/activate + +After this, they'll need to check-out the code from code repository. Assuming it is a git repo this will just be a *git clone* command. + + git clone + +Now navigate to this sub-folder. + + (venv) user@machine $ cd myproject/myrepo + +Install the dependencies: + + pip install -r requirements.txt + +That's it! Your developer machine is now all set for coding. + +This page is an excellent reference for virtualenv: https://python-guide-pt-br.readthedocs.io/en/latest/dev/virtualenvs/ for anything virtualenv. + +Hope this post has been useful to you. + + + diff --git a/_posts/2017-6-16-pretty-print-json-with-command-line.md b/_posts/2017-6-16-pretty-print-json-with-command-line.md new file mode 100644 index 0000000000000..9abf7fec40dc6 --- /dev/null +++ b/_posts/2017-6-16-pretty-print-json-with-command-line.md @@ -0,0 +1,105 @@ +--- +layout: post +title: Pretty Print JSON & Move it to Command Line +comment: true +description: Pretty printing JSON is a common operation. I show how to get this done and set it as a command. +image: /images/blog/command_prompt.png +tag: [json,pretty-print] +--- +Pretty printing JSON is a very common operation. In this post, I show how to build the code and make the script an executable as a first-class command. + +![Command Prompt](/images/blog/command_prompt.png) + +## Building Pretty Print program +In python, pretty-printing is very straight forward. It only needs the use of the built-in library __json__ and using the __dumps__ method. Given any data, the following code can format and sort the keys of JSON and print it to console. + +```python +import json +json.dumps(json_data, indent=4, sort_keys=True) +``` + +## Creating work-flow +To this single line, we need to add the argsparse code so that it can take input parameters, and add a help function as well. + +In this code I have added the ability to: + +- specify the JSON file that is the input +- specify an optional output file that will contain the formated JSON. By default, we over-write the incoming JSON file. + +That's pretty much it. There is not much of error handling since we don't really need a lot of it required. + +The code is available here: https://github.com/akshayranganath/python_pretty_printer. + +```python +#! /usr/bin/env python +############### +# Author: akshayranganath +# Gihub link: https://github.com/akshayranganath/python_pretty_printer +############### +import json +import argparse + +def getFileData(fileName): + """ + Open a file, and read the contents. The with..open operation will auto-close the file as well. + """ + with open(fileName) as handle: + data = handle.read() + + return data + +def prettyPrint(data, outfile): + """ + Pretty print and write the file back to the argument 'outfile' + """ + with open(outfile, "wb") as handle: + handle.write ( json.dumps(json.loads(data), indent=4, sort_keys=True) ) + + +if __name__=="__main__": + parser = argparse.ArgumentParser(description='Pretty print JSON') + parser.add_argument('--file', help="JSON file. If no --outfile is provided, this file will be over-written",required=True ) + parser.add_argument('--outfile', help="Output file to pretty print the JSON", required=False ) + + args = parser.parse_args() + outfile = args.file if args.outfile is None else args.outfile + + jsondata = getFileData(args.file) + prettyPrint(jsondata, outfile) + print 'Pretty printer complete.' +``` + +## Converting to command line +Converting this script to a command line is a two step process: + +- add execute permission +- move code to a folder in the shell PATH + +### Add execute permission +To give execute permission to the script, simple run this command: + +```bash +chmod +x pretty_printer.py +``` + +This will provide execute access only for the user running the command. If you want all users on this system to have the execute permission, issue this command instead: +```bash +chmod a+x pretty_printer.py +``` + +### Making script a command +To make the script work like a regular command, move it to a location that is on the shell's __PATH__ list. On Linux/Mac, you can get this with the __echo__ command + +```bash +echo $PATH +``` + +On Windows system, execute the _set_ command and look for _PATH_. + +Based on the output, move the pretty_printer.py to a location specified in the PATH directory list. For Linux/Mac users, you could move it to _/usr/local/bin_. Once done, you can execute this as a built-in command from any folder. + +After it is setup correctly, you can execute it like this. +```bash +$ pretty_printer.py --file rules.json +Pretty printer complete. +``` diff --git a/_posts/2017-6-23-Git-Pull-Handling-Merge-Conflict.md b/_posts/2017-6-23-Git-Pull-Handling-Merge-Conflict.md new file mode 100644 index 0000000000000..b87feaf53502b --- /dev/null +++ b/_posts/2017-6-23-Git-Pull-Handling-Merge-Conflict.md @@ -0,0 +1,92 @@ +--- +layout: post +title: Handling a Git Pull request with merge conflict +comment: true +description: Merging a git pull request with conflicts is not straight-forward for the first time. Here's how to do it. +image: /images/blog/pull_request.png +--- + +When working with Git, the relatively complex tasks are issuing a pull request and then merging with conflicts. Due to the prevalence of UIs, pull requests are now quite simple. However, merge requests that have a conflict are a little bit more hard to handle. Here's how I get this done. + +![pull request](/images/blog/pull_request.png) + +In this example, let's work with 2 branches: + +- prod (the final source of truth) +- feature-1 (the branch issuing pull request) + +## Step 1: Verify your local repo +To start off, ensure that you have the latest files for the __prod__ branch. + +```bash +git checkout prod +git pull origin prod +``` + +This will ensure that the files on local repository are in-sync with your remote git repo (Github/Bitbucket, etc). + +## Step 2: Switch to branch +The next step is to switch to the branch that you want to merge. While doing it, ensure that you also pull the latest files from your remote server. + +```bash +git checkout feature-1 +git pull origin feature-1 +``` + +## Step 3: Try to merge +At this point, we have the latest files for both the "prod" and "feature-1" branch locally. You are also on the branch that needs to be merged. Now try this branch to your target branch/master. + +```bash +git merge prod +``` + +You should see output like this: +```bash +Auto-merging origin_settings.py +CONFLICT (content): Merge conflict in origin_settings.py +Auto-merging aggregator.py +Automatic merge failed; fix conflicts and then commit the result. +``` + +So git is saying that: + +- File _aggregator.py_ has some change but, it can be merged with no conflict. +- File _origin_settings.py_ has some changes that overlap. There is a merge conflict and it cannot automatically merge the change. + +Git would have also tried to merge the files and added the conflict related information to the file that has issues. In this case, it is a file named _origin_settings.py_. + +## Step 4: Resolve the merge conflict +To resolve the conflict, open the file and look for lines like this: + +```python + data = open('cert2.txt').read() + cert = TLSCertificate() + <<<<<< HEAD + print cert.loadCertificate(data) + ====== prod + # print "hello world" +``` + +In this case, git says that the line to print _"Hello world"_ from _prod_ branch was over-written with a different print statement in the branch. Manually, you'll need to fix this. So in my case, I fixed it this way: + +```python + data = open('cert2.txt').read() + cert = TLSCertificate() + print cert.loadCertificate(data) +``` + +### Step 5: Check in changes +Now, commit the fixes to the branch. + +```bash +git add origin_settings.py +git commit -m "some comment" +git push origin feature-1 +``` + +### Step 6: Merge the branch +At this point, the conflicts should be gone. You can now log in to your remote server like github or bitbucket and hit the merge request. Once this completes, the pull request is automatically marked as complete. + +When completing the merge, there is an option to close the branch as well. Use this option if you feel that the branch created is no longer necessary. + +Hope you find this useful! \ No newline at end of file diff --git a/_posts/2017-6-30-Fix-Mixed-Content-Issue-CPS-Report-Only.md b/_posts/2017-6-30-Fix-Mixed-Content-Issue-CPS-Report-Only.md new file mode 100644 index 0000000000000..5b44e1acc79b4 --- /dev/null +++ b/_posts/2017-6-30-Fix-Mixed-Content-Issue-CPS-Report-Only.md @@ -0,0 +1,86 @@ +--- +layout: post +title: Fix mixed content issue with Content Security Policy report +comment: true +description: Mixed content can cause browser warnings. Finding such pages can be automated by using Content-Security-Polciy reports and I show how to do this. +image: /images/blog/needle.jpg +--- + +One of the biggest challenges of migrating a website from HTTP to HTTPS is the issue of mixed-content warning. I had written on [how content-security-policy (CSP) can help in this HTTPS migration process](https://akshayranganath.github.io/csp-to-avoid-mixed-content/). However, one of the more fundamental problem is finding pages that have a mixed content issue. + +![needle in haystack](/images/blog/needle.jpg) + +## What is the challenge? +When a webmaster wants to move a website from HTTP to HTTPS, the typical process is: + +1. Enable HTTPs on all the sub-domains and the primary domain. +2. Add "[Content-Security-Policy: upgrade-insecure-requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/upgrade-insecure-requests)" +3. Find and fix mixed content warnings on all the pages +4. Start issuing a default HTTP to HTTPS redirect. There may be some SEO impact. Do refer to this excellent blog on [301 redirection for SEO](https://moz.com/blog/301-redirection-rules-for-seo). +5. Finally, enable Strict-Transport-Security ([HSTS](https://www.owasp.org/index.php/HTTP_Strict_Transport_Security_Cheat_Sheet)) on all the domains + +Step #3 could be especially hard for very large websites. Finding pages that have a mixed content issue is a deal breaker as well. If it is not done, then the browser will show the warning on console and disable the padlock icon as well. + +One of the ideas we had was, "What happens if we enable both the 'upgrade-insecure-requests' and HSTS headers?" That way, even when the base HTML references a resource as 'http', the other directives will force it be requested over HTTPs. Although this is true, at least in Chrome, the browser first flags the warning on mixed content and breaks the padlock first. The directives to convert to https seems to occur only when the actual resource is requested. This research was done by [Paul Calvano](https://twitter.com/paulcalvano) and posted the results on an internal forum at Akamai. + +So the only solution to ensure that you do migrate successfully is to actually fix all insecure references to https. + + +ALERT!
"Content-Security-Policy: upgrade-insecure-requests" is not available in IE browsers. Users on this browsers will not only get the warning but, the requests to embedded objects will never be promoted to HTTPs as well. + + +## Enter Content-Security-Policy Report +Content-Security-Policy offers 2 flavors of implementation: + +- [Content-Security-Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP) to actually implement specific access restrictions +- [Content-Security-Policy-Report-Only](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only) to simulate the implementation of a policy and report on the impact. + +For this blog post, our interest is in the reporting feature of CSP. The reporting can be setup by having an end-point defined. It could be on the same domain or on a completely separate domain. If you a website with hundreds of requests per day, you could consider using the free service at [https://report-uri.io/](https://report-uri.io/). + + Content-Security-Policy-Report-Only: default-src https: ; report-uri /csp-violation-report-endpoint/ + +Once it is setup, access violations are reported using a JSON format. So you'd see a report like this for a specific policy violation. + +```json +{ + "csp-report": { + "document-uri": "https://example.com/signup.html", + "referrer": "", + "blocked-uri": "http://example.com/css/style.css", + "violated-directive": "style-src cdn.example.com", + "original-policy": "default-src 'none'; style-src cdn.example.com; report-uri /_/csp-reports", + "disposition": "report" + } +} +``` + +Here's what this means: + +- Error was fired on the base page at _https://example.com/signup.html_ +- The URL blocked was _http://example.com/css/style.css_ +- This http object was references in a _style-src_ object, ie a _link rel="stylesheet"_ + +## How does reports help? +When the reporting is setup, an alert is fired for each of the blocked request. By building some tooling, it would be very easy to generate a report providing a list of blocked URLs and the base page where the resource was blocked. So you could have something like this: + +| Base Page | Blocked Resource | Referenced as | +| --------- | ---------------- | ------------- | +| /signup.html | /css/style.css | style-src | + +As a webmaster, you could then share it with the developer team. They could reference the base page tab and then fix the insecure reference. The development team could then change any __http://__ references to be relative reference __//__. This way, there is no dependency on forcing https. The embedded page will use the same protocol as the base page in such cases. + +### Progressively Add Security +Since there are browser inconsistencies, one of the better ways to proceed with full site HTTPS migration is to use both CSP and CSP report-only headers. CSP header should perform the __upgrade-insecure-requests__. This will occur for all browsers other than IE. You can also use CSP report only to get a report of blocked URL. In this way, you'd get: + +``` +Content-Security-Policy-Report-Only: default-src https:; report-uri /csp-violation-report-endpoint/ +Content-Security-Policy: upgrade-insecure-requests; +``` + +- full site HTTPS behavior on browsers that do support upgrade-insecure-requests +- warning and mixed content messages that can be used to clean up the code. + +There is no issue with using both headers. + +## Conclusion +Migrating from HTTP to HTTPs could be challenging but, using the right set of CSP headers could help you make this transition a bit more easy. Unfortunately, the only clean way to get this done is to fix the references to any http request to use https. \ No newline at end of file diff --git a/_posts/2017-7-19-how-to-learn-and-adapt-to-devops.md b/_posts/2017-7-19-how-to-learn-and-adapt-to-devops.md new file mode 100644 index 0000000000000..fc31743ba487f --- /dev/null +++ b/_posts/2017-7-19-how-to-learn-and-adapt-to-devops.md @@ -0,0 +1,92 @@ +--- +layout: post +title: How to learn & adapt DevOps? +comment: true +description: Moving from traditional development models to DevOps is a significant transformation. I share my experience on this journey and the resources I used. +image: /images/blog/cocoon.jpg +tags: [devops, automation, automated unit test, git, git hooks, webhooks, transformation] +redirect_from: + - /how-to-learn-and-adopt-to-devops/ +--- + +DevOps is the latest buzzword but, it is also a philosophy that has a potential to change the way organizations build product and services. Being in a professional services role associated with CDN, I was far removed from the actual development teams and had lost touch with my coding background. Over the last 6-8 months, I have been working on to learn about the concepts of this new methodology and felt that there are may be people out there embarking on a similar journey. If you are one of them, hope my approach can give some ideas. Of course, I am still learning - so if you have any thoughts, please let me know! + +![transformation / metamorphosis](/images/blog/cocoon.jpg) + +## tl;dr; +If you want to quickly get an idea on the journey, here's my path to DevOps: + +- Read the books *Phoenix Project* and *The Goal* +- Take an introductory course on DevOps. There are a few of them from Linux Academy, Linda.com, Udacity and Coursera. +- Read the book DevOps Handbook to get a firm foundation +- Build a simple code pipeline with Github. This involves: + - picking up a programming language, + - working with git + - writing unit tests + - adding git hooks and github webhooks + +Basically here's the mapping: + +- DevOps what: Introductory course from Lynda.com, Linux Academy, Udacity or Coursera +- DevOps why: Read Phoenix Project and the Goal +- DevOps how: DevOps hand book, GitHub coding and creating web hooks + +## In the beginning.. + +![sunrise - beginning](/images/blog/sunrise.jpg) + +Before starting on the DevOps journey, let me outline my experience. I used to be a Java/J2EE developer and my last project in this avatar was building a solution over the [liferay](https://www.liferay.com/) portal server. We had a team of 20 and the process was pure waterfall/iterative. I had no background in the world of agile and never attended the scrum sessions, nor understood the user stories. It was like I had missed out on a whole era of software programming. + +In my role, I mainly work on consulting with our customers on various aspects of web performance. Since I worked on Akamai solutions and guidance to customers, I was far removed from the world of actual coding for almost 8 years. About 1.5 years ago, a slow trickle of questions and ideas started to crop up about this devops thing. Over the last few months, it was nearly impossible to escape hearing about the miracles of this elixir. That is when I decided to take a plunge and start to learn about the changes happening in the landscape of development and operations. + +## Laying the Foundations + +As with most people, I had heard a lot about the cloud providers - Amazon AWS, Google Cloud Services and Azure coupled with names like ghithub, chef, puppet and so on. I had assumed all of these amorphous stuff fit into a neat thing that magically transformed into DevOps. And I knew this understanding was totally wrong. + + +So the first thing I realized is I needed a solid footing and gain an understanding of the concepts. Our company had a tie-up with Linux Academy and I found their course title **[DevOps Essentials](https://linuxacademy.com/devops/training/course/name/devops-essentials)** to be very useful. It gave me a gentle introduction on what is devOps as a technical practice and how the commonly heard tools fit in. + +### Theory on DevOps +Now that I had a rough idea on the **what** of DevOps, I wanted to get the reason on **why** this had become so important. As I was searching around, I found the book **[The Phoenix Project](https://www.amazon.com/Phoenix-Project-DevOps-Helping-Business/dp/0988262509)**. It introduced the concept of the pains that a typically organization feels in delivering software and walks through the transformation as a high paced novel. I am still a big fan of this book and would highly suggest this for anyone coming from an IT Service background. I did collect a [list of quotes](https://akshayranganath.github.io/quotes-from-book-the-phoenix-project/) from this book. + +Within the *Phoenix Project* book, there were a lot of references to an earlier book called **[The Goal](https://www.amazon.com/Goal-Process-Ongoing-Improvement/dp/0884271951/ref=sr_1_1?ie=UTF8&qid=1500502236&sr=8-1&keywords=the+goal)**. This was a book written in the manufacturing transformation era of the 80s and spoke about how to optimize the assembly line. Since it was referenced so heavily, I decided to pick this up - and I was amazed. The parallels between code development and assembly line manufacturing was spot on. It was one of those moments when you feel like "I could have thought of that!" And yet, it had taken over 20 years of painful lessons in bad software management to arrive at this wisdom. My point is: If you'd like to understand the fundamentals of devops better, you should read this book and learn about the [theory of constraints](https://en.wikipedia.org/wiki/Theory_of_constraints). + +After reading these 2 books, I realized that there were so many things that I was doing in my previous life that resonated with the hard-ships faced by the protogonists in the novels. We used to have constant delays, missed features, a Javascript pop-up saying "Hi" that made it to production and a lot of other horrendous code management and people management issues. We seemed to have no control. And yet, we would fill out a load of documents and be certified at the highest levels of Capability Maturity Model (CMM) ratings! The cynic in me had finally found some answers! + +## Getting involved.. + +![hands on ](/images/blog/handson.jpg) + +The next step was to get a more in-depth view and learn the **how** of devops. Actually, this step gaining more understanding of the _what_ and _why_ as well. For this purpose, I would highly recommend the book **[The DevOps Handbook](https://www.amazon.com/DevOps-Handbook-World-Class-Reliability-Organizations/dp/1942788002/ref=pd_lpo_sbs_14_t_0/140-1510501-0325548?_encoding=UTF8&psc=1&refRID=J9PM2SX07X936SY6DHJ9)**. It explains in a very clear manner on how organizations can approach the devops transformation. The authors cover various aspects from culture, the tooling and the process necessary to make this transformation. I have published a [collection of quotes from DevOps handbook](https://akshayranganath.github.io/quotes-from-devops-handbook/) as well. + +Apart from the theory, I felt that I needed to understand the various things tools and techniques necessary that leads up to a devOps organization. + +### Gaining Hands-On Experience +To build a more hands-on experience, I decided to actually build some software and adopt the methods being taught by the books. So I used the following things: + +| Purpose | Tool / Technique | Comments | +| ------- | ---------------- | -------- | +| Programming | Python | I was already familiar with python and it just made sense to use it. I had lost touch with Java over the past decade. | +| Environment creation | Virtualenv | [Creating virtual environments](https://akshayranganath.github.io/How-to-create-dev-environment-for-python/) is essential to provide the exact same foundation for code development. So I had to learn the use of virtual environment. Using virtual environment and the command ```pip freeze```, I was able to generate the dependency list for my project. Once this became part of github project, my code could be easily deployed and worked with no surprises. | +| Code repository | Github / BitBucket | The holy grail of most DevOps initiatives is that everything must be an artifact that is tracked in a repository. This includes the project source code and the code necessary to build environments, the test scripts and the various design documents.
I used Github since it is most famous and most common. For all my private coding efforts, I use BitBucket as they offer a private repository for free users. | +| Automated testing | [python unittest](https://docs.python.org/2/library/unittest.html) | Automation testing is an indispensable part of the DevOps transformation. So I had to learn how to write unit tests using a simple frame-work that is supported by the platform. For my projects, [unittest](https://docs.python.org/2/library/unittest.html) is sufficient.
Theory on DevOps recommends more robust testing like [Acceptance Test Driven Development / Behavioral Driven Development](https://www.agilealliance.org/glossary/atdd/). Since I am just picking up, this was an overkill and I skipped it! | +| Build automation | Bitbucket pipeline | To get a feel for the concept of build automation, I enabled the build pipeline on BitBucket. Basically, it takes my ```requirements.txt``` file and installs all the libraries defined for a virtual python environment. I also defined it to run my automation tests once the environment was created.
This process ensured that was able to check
  • Environment creation was successful
  • Checked in code did not break any pre-defined tests
. | +| Build hardening | [git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) | As I was still learning, I experimented with the use of ```pre-commit`` hook within git. This would ensure that a pre-defined set of tests would run on my local machine *before* the code was committed to be checked in. I wish this had existed long ago and it would have saved so many late night effort on code cleanup! | + +## Standing on shoulder of giants.. + +![standing on shoulders](/images/blog/standing_on_shoulders.jpg) + +My learning effort was based on the amazing work being done in this field by so many authors, writes, bloggers and practitioners. I wanted call out a few of them that you could follow as well: + +- **[Jez Humble](https://twitter.com/jezhumble):** Author of the book and maintainer of eponymous website [Continuous Delivery](https://continuousdelivery.com/), he does post interesting nuggets on twitter. +- **[Gene Kim](https://twitter.com/RealGeneKim)**: Co-author of the books Phoenix Project and DevOps Handbook, he is a very good person to follow on twitter. +- **[Martin Fowler](https://twitter.com/martinfowler)**: A veteran at ThoughtWorks, his blog at [MartinFowler.com](https://www.martinfowler.com/) has very good resources and concise definition on the concepts of DevOps. +- **[DevOps.com](https://devops.com/)**: I recently started to follow the blogs from this website. They are quite well written and a good resource to glance through. + +I am still collating good resources. If you have any suggestions, please let me know! + +## In the end.. +At the culmination of my effort, I was able to put out a strategy for adopting DevOps with Akamai. If you are interested, head over to my post [Planning a DevOps Strategy](https://developer.akamai.com/blog/2017/07/11/planning-devops-strategy/). + +That's it for this edition. I hope to take this forward from my learning over the next few months. Hope this has been useful. \ No newline at end of file diff --git a/_posts/2017-7-19-how-to-learn-and-adopt-to-devops.md b/_posts/2017-7-19-how-to-learn-and-adopt-to-devops.md new file mode 100644 index 0000000000000..d5496abccd772 --- /dev/null +++ b/_posts/2017-7-19-how-to-learn-and-adopt-to-devops.md @@ -0,0 +1,10 @@ +--- +layout: post +title: How to learn & adopt DevOps? +comment: true +description: Moving from traditional development models to DevOps is a significant transformation. I share my experience on this journey and the resources I used. +image: /images/cocoon.jpg +tags: [devops, automation, automated unit test, git, git hooks, webhooks, transformation] +redirect_to: + - /how-to-learn-and-adapt-to-devops/ +--- \ No newline at end of file diff --git a/_posts/2017-8-25-DevOps_Using_Python_logging_for_better_reporting.md b/_posts/2017-8-25-DevOps_Using_Python_logging_for_better_reporting.md new file mode 100644 index 0000000000000..a28339b2a9d21 --- /dev/null +++ b/_posts/2017-8-25-DevOps_Using_Python_logging_for_better_reporting.md @@ -0,0 +1,46 @@ +--- +layout: post +title: Using Logging in Python for better reporting +comment: true +description: By using a multi-level logging built into the code can help in making it more DevOps friendly. Python's logging frame-work is a good way to start. +image: /images/blog/satellite.jpg +tags: [devops, automation, automated unit test, logging] +--- + +When moving into an agile / DevOps model of code production, one of the core concepts is building in enough monitoring and telemetry. To enable this, most programming languages come with bundled libraries for logging. There is log4J in Java and [logging](https://docs.python.org/3/library/logging.html) library in python. In this (short) post, I walk through a use case of how it helped. + +![satellite telemetry](/images/blog/satellite.jpg) + +## Nikon RAW Images +I am trying to be a better digital photographer and the first step was to enable shooting in RAW mode. Since I am still new, I shoot in a combined mode. My camera produces a raw _(.NEF)_ and a jpeg _(.jpg)_ image for every shot. When I open the images, it's easy to view the photos and I delete the obviously bad ones from within the tool. So this has a challenge of deleted .JPG files but orphaned .NEF files. + +So I wrote to script to clean up such files. Here's the code: [https://github.com/akshayranganath/FileCleaner](https://github.com/akshayranganath/FileCleaner). + +However, in my first version of the code, I wasn't sure on how best to display the messages. For example: + +- Should I print the name of all the files that would be deleted? +- Should I hide everything and just say "Done"? +- How to signal there is an error as against a message? + +## Enter logging +This is when I realized that logging would be a better solution. It offers varying level of verbosity and an ability to express a message _and tell the kind of message_. So I can have an *informational* message or a *debug* message or an *error* message. {For the full list, refer to [Logging Levels](https://docs.python.org/2/library/logging.html#levels)}. When a user fires the code with a specific verbosity level, I can fine tune the amount of chattiness of the code. + +So if I just want to get the work done, I can invoke this command: +```bash +python file_cleaner.py --folder /some/folder +``` + +If I want to see all the messages including each file being deleted, I can use: +```bash +python file_cleaner.py --folder /some/folder -v +``` + +I could even start to accept specific verbosity level as a number between 0-100 and the messages would vary. + +## Why is it useful? +Here's why I think this is super-cool: + +- As a developer, we often put in print statements. We either comment or delete them as time goes by. However, we'd put this in because we'd been stuck at this point for some reason. By remove the comment, we take away this ability to debug. Instead of using a print, what-if there was a logging.debug() message? This way, the message is never displayed in normal operation but printed out when necessary. +- Logging level could be externalized through an environment variable. This way, when code is deployed, the automation deploying the code could set the verbosity level depending on the complexity of the change and then tweak it over time. So all new code could start at DEBUG level and then move to INFO and then to ERROR. + +I am still learning the usage. So if you have other use cases, do put in a note! \ No newline at end of file diff --git a/_posts/2018-01-05-Accelerate-Some-Quotes-From-The-Book.md b/_posts/2018-01-05-Accelerate-Some-Quotes-From-The-Book.md new file mode 100644 index 0000000000000..6a41fdf1f9185 --- /dev/null +++ b/_posts/2018-01-05-Accelerate-Some-Quotes-From-The-Book.md @@ -0,0 +1,68 @@ +--- +layout: post +title: Acceleate - Quotes from the Book +comment: true +description: Accelerate by Nicole Forgsen and Jez Humble is the book that explains the science behind the State of DevOps reports. It's an amazing read and I highly recommend it for anyone trying to start out or fine-tuning their DevOps journey. +image: /images/blog/Accelerate_3D-300x300.jpg +--- + +[Accelrate - Building & Scaling High Performing Technology Organizations](https://itrevolution.com/book/accelerate/) is a book by [Nicole Forgsen](https://twitter.com/nicolefv) and [Jez Humble](https://twitter.com/jezhumble). It takes on the journey behind the [State of DevOps](https://puppet.com/resources/whitepaper/state-of-devops-report) reports. + +![Accelerate book cover](/images/blog/Accelerate_3D-300x300.jpg) + +I had received an early edition at the DOES-San Francisco and managed to read it over the holidays. It is a very well written book that explains the trends and the patterns seen in DevOps for past 3-5 years. I feel that it is very useful for organizations and practitioners who are helping their teams navigate the DevOps transformation. It can help set better goals and hint at the kind of organizational structure and reporting mechanisms needed to achieve the promises of DevOps. + +It also satisfies the nerd in me. The detailed explanation of the methodology, the background to clustering and the extended bibliography is pretty intese and exciting read. + +Here are a few of the quotes that I wrote down for my reference. And I highly recommend the book! + +## Quote from the book + +Queue theory in math tells us that as utilization approaches 100%, lead time approaches infinity - in other words, once you get to very levels of utilization, it takes exponentially longer to get anything done. + +Successful measure should: + +- focus on a global outcome and not on local outcomes +- focus on outcomes and not outputs (to avoid rewarding people for busywork) + +### DevOps metrics + +1. Delivery lead time: time it takes to go from code committed to code successfully running in production +2. Deployment frequency +3. Mean Time to restore service +4. Change fail rate + +Westrum provides 3 characteristics of good information: + +1. It provides answers to the questions that the receiver needs answered +2. It is timely +3. It is presented in such a way that it can be effectively used by the receiver + +### What gets you to Continuous Delivery? + +- version control +- deployment automation +- trunk-based development +- test automation +- test data management +- shift left of security +- loosely-coupled architecture +- empowered teams +- monitoring +- proactive notifications + +__Change Approvals__ + +.. approval by an external body __simply doesn't work__ to increase the stability of production systems.. it certainly slows things down. + +__Team Experimentation__ + +.. if a development team isn't allowed to change requirements or specifications in response to what they discover, without authorization from some outside body, their ability to innovate is sharply inhibited. + +__Failure Modes__ + +Managers should also watch for other contributing factors, and remember that human error is never the root cause of failure in systems. + +__Successful DevOps Transformation__ + +Though we often hear stories of DevOps and technology-transformation success coming from the grassroots, it is far easier to achieve success when you have leadership support. diff --git a/_posts/2018-08-16-Book-Review-The-Perfect-Weapon-David_Sanger.md b/_posts/2018-08-16-Book-Review-The-Perfect-Weapon-David_Sanger.md new file mode 100644 index 0000000000000..a58bc8fbdaa69 --- /dev/null +++ b/_posts/2018-08-16-Book-Review-The-Perfect-Weapon-David_Sanger.md @@ -0,0 +1,36 @@ +--- +layout: post +title: Book Review on "The Perfect Weapon By David Sanger" +comment: true +description: David Sanger's book, the Perfect Weapon raising some thorny issues on cyber warfare along with an great narration of some incredible hacks that went on from Stuxnet to the audacious DNC hacks. +image: /images/blog/hacker-perfect-weapon.jpg +--- + +I recently completed the book _The Perfect Weapon: War, Sabotage, and Fear in the Cyber Age_. I first heard about it on an NPR show, [All Things Considered](https://www.npr.org/2018/06/14/620103358/the-perfect-weapon-tells-the-story-of-growing-cyber-war-that-the-u-s-is-fighting). The author was being interviewed about the book. He described the audacious hacks of US-Israeli intelligence in the Stuxnet and moved on to more imminent threats and the issues of cyber security. + +After reading this line from the cover of the book, I was hooked! + +>The Perfect Weapon is the startling inside story of how the rise of cyber-weapons transformed geopolitics like nothing since the invention of the atomic bomb. Cheap to acquire, easy to deny, and usable for a variety of malicious purposes — from crippling infrastructure to sowing discord and doubt — cyber is now the weapon of choice for democracies, dictators, and terrorists. + +Amazon Book Review + +So I managed to get one of the first editions of the book from the local library and read it cover-to-cover. It starts off from the story on how the author broke the story about Iran's centrifuges spinning of control and the hand of US in this effort. It then dwelt on the story of North Korea's missles simply falling off the sky for no apparent reason and some informed speculations about the cause. Pulling in from an eclectic set of sources the author guides us through the recent wave of cyber attacks, it's impact and the problems of retaliation. You see, the world of "cyber" is a whole new realm and the laws of warfare are not clear. + +![Hacker](/images/blog/hacker-perfect-weapon.jpg) + +Apart from the astounding stories of hacks by US, China, Russia and Iran, the book is worth for the pertinent questions it raises. Traditional wars has a set of loosely followed rules like the [Geneva convention](https://en.wikipedia.org/wiki/Geneva_Conventions), concepts of aid workers not being harmed and clearly displaying affiliation. Even during the Cold War, chilly concepts like [Mutually Assured Destruction](https://en.wikipedia.org/wiki/Mutual_assured_destruction) existed. Due to this, there were policies around how the cold war actors would escalate the war and what would be off-limits. However, the world of cyber warfare has no such rules. + +To me, these points were stood out: + +- The reason we have some clarity on regular warfare and nuclear warfare is the relative transparency. Armies have tactics of not harming civilians. Nuclear Nations have rules of engagement and esclation. However, cyber warfare has no limits and boundaries. +- Traditional warfare had clear actors. In the world of bits and bytes, attribution is hard. Even when it is accomplished, proving the link from an action to an event could be difficult. +- Cyber warfare is a bit of enigma for even the nation's forces. On the one hand, they use it for offense but, the same group can never assure it's own resources are protected. Vulnerabilties and trojans can lay buried for many years with no seeming symptom of infection. +- Engaging in a policy discussion becomes a zero sum game. To start a discussion, a nation will have to disclose the kind of attacks it is capable of. By exposing this, the other actors can immediately take corrective action and close the loop-hole. So there is no motivation to disclose the tactics and not much of hope on starting a negotiation. +- Cyber wafare is asymmetric. Actors don't need to spend billions on research or build large armies. Penetrating enemy's system does not take a very large number of cyber warriors and it is relative cheap. +- Unlike traditional warfare, cyber warfare can be carried out in the realm of civilan groups. This makes it harder to retaliate. For example, if a so called activist group in Russia attacks US, it cannot consititute as an act of war. So the US government may not be able to openly call out and retaliate against the attack. +- The relationship of "warriors" and the government gets complicated. In the era of Cold War, the defense department could call for bids to make a missile with a clear intention of building weapons. However, in the world of cyber, the technology generally cross pollinates from private companies to defence purposes. So a drone software used to detect building measurements can be adapted for anti-terrorist strikes. However, the authors could choose not to supply the technology to the government. +- Finally, the convergence of full encryption, quantum computing and AI could open up a whole can of worms making the discussion even more murky and difficult. + +The bottom line, there is a very high chance that we can end up with accidentally triggered cyber warfare that gets escalated beyond control and with repurcussions that goes far beyond the planned action. + +It is a chilling thought and something worth pondering over... \ No newline at end of file diff --git a/_posts/2018-10-11-Python-Stats-From-Jenkins-Job-Output.md b/_posts/2018-10-11-Python-Stats-From-Jenkins-Job-Output.md new file mode 100644 index 0000000000000..df02130512da8 --- /dev/null +++ b/_posts/2018-10-11-Python-Stats-From-Jenkins-Job-Output.md @@ -0,0 +1,104 @@ +--- +layout: post +title: Python Stats from Jenkins Job Output +comment: true +description: Jenkins jobs could contain some script output that is archived. Using python statistics, I show how you could extract and create summarized results. +image: /images/blog/math.jpg +--- + +When working with Jenkins, you may be running some jobs that provides some kind of metric. For example, on a website, you may be monitoring the page load time at every hour, the median / 90th percentile CPU load time, etc. If you are running this as a Jenkins job, the output may be stored as a flat file, JSON or some such format. Typically, this is dumped into the `archive` folder. In this blog post, I show how to extract the data and get some meaningful metrics. + +![Math](/images/blog/math.jpg) + +## Initial Setup +In this case study, I am using a website performance information. Specifically, I am pulling down the median `page load time` for this blog. The data is pulled down at every hour and stored in a flat file called `summary.json`. The format of the JSON file is as follows: + +```javascript +{ + "p98": "2611", + "median": "2611", + "p95": "2611", + "moe": "0.0", + "n": "1" +} + +For this workflow, we only care about the `median` metric. We could as well swap the metric and compute our data for other percentiles as well. + +``` +I am actually using the Query API for mPulse. I'll have a follow up post on the exact work-flow. + +This Jenkins job runs every hour. Jenkins then moves this JSON file to a folder like `$JENKINS_PATH/jobs/$JENKINS_PIPELINE/builds/$BUILD_NUMBER/archives/summary.json`. + +## Computing Variance +The next step is to use this `summary.json` that is generated every hour to run a check: +__Is the current performance any different from the past performance?__ + +In this use case, _past performance_ is determined by three parameters: + +- median / mean of the past 20 runs +- _variance_ of the past 20 runs +- _standard deviation_ of the past 20 runs + +If you'd like to brush up the stats, please refer to this _MathIsFun_ page on [Standard Deviation, Variance and more](https://www.mathsisfun.com/data/standard-deviation.html). + +Here's my algorithm to compute the moving performance benchmark: + +* Find all the Jenkins job output folder +* Sort the job folder by their numeric job number. For this I had to adopt the logic dicussed in [Human Sorting](https://nedbatchelder.com/blog/200712/human_sorting.html) blog post by Ned Batchelder. +* Within the archive folder, read the `summary.json` and pull out the value for the node named `median` +* Store this value into an array. +* After completing this process, extract the last 21 values ignore the latest. Basically take an array splice `[-21:1]`. +* Compute the standard deviation for the last 20 values. +* If performance from the latest run is greater than or lesser than 1 standard deviation away, alert this as an outlier. + +### Code Snippets + +The first part I described earlier is to read the `summary.json` and extract the median metric. We can do this with this kind of code. + +```python +jobs_path = $JENKINS_PATH+$JENKINS_PIPELINE+'mPulse/builds' +directories = os.listdir(jobs_path) +directories.sort(key=natural_keys) +#print(directories) +#/var/lib/jenkins +page_load_times = [] +for each_directory in directories: + each_job = jobs_path + '/' + each_directory + if os.path.isdir(each_job)==True and os.path.islink(each_job)==False: + #print(each_directory) + #check for summary.json + each_summary_file = each_job + '/archive/summary.json' + if os.path.isfile(each_summary_file): + with open(each_summary_file) as f: + data = json.loads(f.read()) + if 'median' in data and data['median']!=None: + page_load_times.append(int(data['median'])) +``` + +The next step is to get the last 20 values, without the latest run. + +```python +last_20_values = page_load_times[-21:-1] +``` + +Next up is to compute the stats. +```python +median = statistics.median(last_20_values) +stddev = statistics.stdev(last_20_values) +variance = statistics.variance(last_20_values) +``` + +Finally, compare and find anomaly. +```python +if (last_median < (median - stddev)) or (last_median > (median + stddev)): + print("***** ALERT ********") + print("Performance anamoly detected: " + str(last_median) ) + #force a build failure on jenkins + sys.exit(1) +``` + +In this case, I am failing the build when the current median is more than 1 standard deviation away. This will give me an easy visual indication of error. + +By doing this, you now have an easy to use "Anamoly detection" code! If you want to make it more interesting, we could replace the simple 1 standard deviation rule by more involved computation like the [Nelson's Rule](https://en.wikipedia.org/wiki/Nelson_rules). + +Hope this has been fun! I'll have a follow up blog that explains the entire Jenkins pipeline and the demo of mPulse API that was used to extract the raw information. diff --git a/_posts/2018-12-20-Understand_SPA_and_Tracking_SPA_pages_with_RUM.md b/_posts/2018-12-20-Understand_SPA_and_Tracking_SPA_pages_with_RUM.md new file mode 100644 index 0000000000000..1bcae1d951477 --- /dev/null +++ b/_posts/2018-12-20-Understand_SPA_and_Tracking_SPA_pages_with_RUM.md @@ -0,0 +1,70 @@ +--- +layout: post +title: Quick Primer to Single Page Application (SPA) and Tracking SPA pages +comment: true +description: Understand the history API that powers Single Page Application (SPA) and learn how this is leveraged by real user monitoring solutions like mPulse for reporting performance of SPA pages . +image: /images/blog/mobile-phone.jpg.jpg +--- + +## tl;dr; +SPA relies on HTML 5 History API. Using mPulse's SPA settings, we can track the soft-navigation. Soft-navigation is when the URL changes on the browser but, it doesn't actually re-load the entire page. + + +I was trying to understand the working of Single Page Applications and came across the article [Using the HTML5 History API](https://css-tricks.com/using-the-html5-history-api/) by [CSS-Tricks](https://css-tricks.com/). In this excellent article, they explain the concept of the HTML5 history API. Normally, when you navigate from one URL to another, the browser re-loads the whole page. However, you may want to change certain parts of the page and not re-load the entire page. This could be accomplished by using AJAX requests. However, if you just use AJAX, the URL of the page remains same. So if you need to share this specific view, there is no way to do so.. That is where the History API comes into picture.. + +![mobile](/images/blog/mobile-phone.jpg.jpg) + +## Understanding the History API + +The History API is able to manipulate the URL without forcing the browser to load the entire page. It offers 2 main functions `history.replaceState` and `history.pushState`. + +* `history.replaceState`: This just changes the URL but, does change the navigation flow in the browser. So a user hitting the back button will not go back to original view of this SPA page but, to the previous page from where they came from. +* `history.pushState`: This method will replace the URL _and_ record the current view information. So a user hitting the back button will go back to the original view of the SPA page. + +For more detailed explanation of the 2 methods, refer to the official documentation at [Manipulating the browser history +](https://developer.mozilla.org/en-US/docs/Web/API/History_API) on the MDN site. + +## Measuring performance of SPA navigation + +SPA introduces a challenge to measuring performance. On regular webpages, browsers are collecting a host of performance related information based on the [NavTiming API](https://developer.mozilla.org/en-US/docs/Web/API/Navigation_timing_API). However, when a SPA navigation occurs, the page doesn't really re-load. Since this does not occur, the browser is not rebuilding the DOM. As a result, it is not able to report on the performance associated with the navigation. + +To work around this, Real User Monitoring (RUM) solutions like [mPulse](https://developer.akamai.com/akamai-mpulse) rely on the history API. By observing the changes to this plugin, it is able to detect a navigation and then measure the timing information. + +## Testing SPA Page + +Since I work at Akamai, I had access to mPulse. So I thought of trying to see how this works. I created a dummy website using the code for [Ghostbusters SPA](https://css-tricks.com/examples/State/) from the CSS-Tricks website. In their example, we have the following: + +* Landing URL: https://css-tricks.com/examples/State/ +* SPA Navigation: + * https://test-woocommerce.akshayranganath.com/peter.php + * https://test-woocommerce.akshayranganath.com/ray.php + * https://test-woocommerce.akshayranganath.com/egon.php + * https://test-woocommerce.akshayranganath.com/winston.php + +In effect, when the user first lands at the home page, it is a regular navigation. This is termed as `SPA Hard` in mPulse. All other navigations within this sub-section of the website is termed `SPA Soft` or soft navigations. If a user were to directly use the URL for any of the character, it would cause the browser to load the page normally and be measured as a `SPA Hard` navigation event. + +### Test Results + +When I setup the tests, the median page load time shows up as 15s based on 8 views. This included both SPA (SPA-Soft) and non-SPA (SPA-Hard) timing information. + +![all beacons](/images/blog/spa_all_beacons.png) + +When I look at the performance of hard navigation, the page load time from 4 views now jumps to 17.4s. + +![spa hard beacons](/images/blog/spa_hard_beacons.png) + +If I now filter to just the soft navigations, the page load time drops to 129 ms from 4 views. + +![spa soft](/images/blog/spa_sort_beacons.png) + +If we never had the soft navigation beacons, we may have mis-interpreted the site performance as over 17s. However, this represents the initial view that includes the time to load SPA framework. However, the pages speed up dramatically once the framework is ready. Without the information on soft navigation calls, you may mis-interpret the engagement on your website. A user in this case could load the home page and access 4 other sub-pages. Without SPA tracking, your RUM analytics may show a user just accessed 1 page and bounced. This could corrupt your bounce rate and conversion measurements. + +### Take-aways + +If you have a SPA based website you could be missing critical performance information signals or mis-interpreting the data. + +* Enable SPA tracking if you have designed your page to use SPA frameworks. +* Understand if you have the right filters in place when looking at numbers. Mixing the SPA and non-SPA navigation will give you numbers that may not be actionable to your teams. +* On other hand, missing SPA navigation will reduce the engagement and conversion metrics. So carefully instrument their measurement for a holistic picture. + +Hope this has been useful. \ No newline at end of file diff --git a/_posts/2018-8-1-3-Trends-in-Web-Performance.md b/_posts/2018-8-1-3-Trends-in-Web-Performance.md new file mode 100644 index 0000000000000..113636ec21b27 --- /dev/null +++ b/_posts/2018-8-1-3-Trends-in-Web-Performance.md @@ -0,0 +1,93 @@ +--- +layout: post +title: 3 Trends in Web Performance +comment: true +description: Web Performance landscape has evolved with focus on perceived performance, RAIL model, focus on mobile website speed and better tooling. The blog explains new metrics and relates them to measurement tools. +image: /images/blog/speed-2018.jpg +--- + +Website performance is always an ongoing effort and an elusive target. In the early days, pageLoad time was the gold standard of web performance monitoring. However, websites have evolved to be a lot more dynamic, Javascript heavy and filled with rich images and 3rd party content. We need better metrics, a better model and better tooling to measure and monitor the performance. In this blog post, I explain the evolution 3 different trends that is shaping the industry: + +- [Response-Animation-Idle-Load (RAIL) model](https://developers.google.com/web/fundamentals/performance/rail): It is both a model and a prescriptive framework that sets up performance benchmarks +- Perceived Performance / [User-centric performance metrics](https://developers.google.com/web/fundamentals/performance/user-centric-performance-metrics): A set of measures that tries to more accurately predict the user's level of engagement / frustration at using a web page. +- Tooling: Set of tools like [CrUX](https://developers.google.com/web/tools/chrome-user-experience-report/), [PageSpeed Insights](https://developers.google.com/speed/pagespeed/insights/), [LightHouse](https://developers.google.com/web/tools/lighthouse/) and RUM solutions like [mPulse](https://www.akamai.com/us/en/products/web-performance/mpulse-real-user-monitoring.jsp) + +So let's dig in! +![Speed](/images/blog/speed-2018.jpg) + +## RAIL Framework +RAIL is a framework that is being publicized by Google. According to the [Google developer page](https://developers.google.com/web/fundamentals/performance/rail) it is + +>a user-centric performance model that breaks down the user's experience into key actions + +The framework breaks down performance into these 4 key goals: + +![RAIL model](https://developers.google.com/web/fundamentals/performance/images/rail.png)Source: https://developers.google.com/web/fundamentals/performance/rail + +| Measure | Explanation | Target | +| ------- | ----------- | ------ | +| Response | Complete a transition initiated by user input within 100ms. | < 50 ms | +| Animation | Aim for visual smoothness. Users notice when frame rates vary | Produce frame < 10 ms | +| Idle | Maximize idle time to increase the odds that the page responds to user input | Response < 50 ms | +| Load | Load pages so that they are interactive quickly based on network / device type | on 3G, time to interactive < 5s | + +Basically, the RAIL model says that load your page fast so that it is interactive quickly. Once it is interactive, ensure that any dynamic aspects are fast enough so that the page remains interactive to user input. + +For more information about RAIL, please watch this video about _[RAIL in real word](https://www.youtube.com/watch?time_continue=3&v=iIV1xPFXmBs)_. + +## Perceived Performance +How fast your site _appears_ to load is as important as it really loads. The problem with perceived performance is that there was no way to measure it. It's something of a conundrum like this: + +![PR Perception cartoon](http://forwardthinkingpt.com/wp-content/uploads/2014/09/24cartoon.jpg) +Source:https://www.cartoonstock.com/directory/p/public_perception.asp + +We have had a lot of measurements like Time to first byte, DOM loaded, onLoad and a host of other measurements. These metrics have been immensely helpful in the evolution of the web performance industry. However, they were designed on the events that could be measured by browsers. However, they failed to answer some basic questions like these: + +| Experience | Meaning | Metric | +| __Is it happening?__ | Did the navigation start successfully? Has the server responded? | First Paint / First Contentful paint | +| __Is it useful?__ | Has enough content rendered that users can engage with it? | +| __Is it usable?__ | Can users interact with the page, or is it still busy loading? | +| __Is it delightful?__ | Are the interactions smooth and natural, free of lag and jank? | + +![Perceived performance metrics](https://developers.google.com/web/fundamentals/performance/images/perf-metrics-load-timeline.png) + +Source: https://developers.google.com/web/fundamentals/performance/user-centric-performance-metrics + +Going beyond this, we have a few more measurements like the following: + +| Experience | Meaning | Metric | +| __Is it slow to interact?__ | How much time did it take from the first user interaction to the page responding to this action? | First Input Delay | +| __Is there a mismatch in expectation?__ | Based on content being loaded, users may try to interact but the page fails to respond. The gap measure this difference. | Time to First Interactive - Time to First Interaction | +| __How frustrated is the user?__ | When the user can't interact, they may first double click and click continuously in the hope of making a page work. | Rage Clicks | + +All this boils down to measuring a host of metrics that is spread across different tools and platforms. + +![What does perception really mean?](https://blogs.akamai.com/blog2pic2.png) +Source: PERCEPTION MATTERS: MAKING RUM MORE REAL + +If you notice, we've now gone from hard measurement of DNS or onLoad to softer measurements related to user's interaction and experience. All this is harder to measure but, Google has announced that their search indexing will now take these factors into account. Specifically, Google will now: + +- use mobile page speed as a ranking signal +- encourage "developers to think broadly about performance" +- and hint at using suggestions from tools like LightHouse, CrUX and PageSpeed Insights which we'll discuss next. + +Source: Using page speed in mobile search ranking + +## Tooling for Perceived Performance + +[WebPageTest](https://www.webpagetest.org/) has been the best and de-facto tool to visualize and show the concept of perceived performance. The film strip view, video and visual progress charts have been the toolset that we have been using for a long time. However, we now have more tools that can help in building a case for measuring and optimizing for perceived performance. + +- [LightHouse](https://developers.google.com/web/tools/lighthouse/): Project LightHouse by Google is able to run a lot of audits and present findings. These findings are wide ranging like providing a filmstrip view to Chrome Javascript timeline. By using the audit, you would be able to dig into problems like a particular Javascript taking high amount of time and thus causing rendering issues. +- [Chrome User Experience Report (CrUX)](https://developers.google.com/web/tools/chrome-user-experience-report/): Google Chrome collects performance measurements from real browsers. This information is sent back to Google, if users have opted in for the data collection. This anonymized data is now being exposed as the Real User Monitoring (RUM) data for Chrome under the project CrUX. This tool is especially helpful since it gives you a spread of the user experience. Synthetic tests can only give you the data from simulated users with pre-defined setup. CrUX can provide histograms of actual performance behaviors, including perceived performance information. +- [Google PageSpeed Insights](https://developers.google.com/speed/pagespeed/insights): This tool has now evolved to provide both suggestions to improve the page and pull up the CrUX data for the URL being audited. By using the PSI report, you should get an idea on the current performance and tips to analyze and fix your website. +- [Akamai mPulse](https://www.akamai.com/us/en/products/web-performance/mpulse-real-user-monitoring.jsp): Akamai's mPulse Real User Monitoring solution provides all the perceived performance data along with the standard metrics like onLoad. The tool can be instrumented to measure rage clicks as well to identify the user's frustration levels when using a website. + +## What do you look for? +By correlating the data from syntehtic tests and the RUM solutions, you should be able to identify potential issues in perceived peformance. Some of the issues could be: + +- synchronous scripts holding up the browser's main thread and preventing it from rendering the content +- A/B testing solutions that implement some fix leading delayed rendering of the page ([Preventing flickering with A/B Solutions](https://growthhackers.com/questions/ask-gh-how-do-you-deal-with-flickering-when-ab-testing/)) +- page _appearing_ to be ready but, actually busy. This can be seen in the first input delay (FID) as well as the difference between Time to First Interaction and Time to First Interactive. +- page _appearing_ to render but not showing the content that matters. This can be caught by metricsl like First Contentful Paint, First Meaningful Paint and Visually Ready + +I have highlighted a very small subset of issues that could be tracked and worked on by leveraging the pereceived performance metrics. If you have any use cases, I'd like to hear more as well! \ No newline at end of file diff --git a/_posts/2019-11-20-How-to-move-images-in-Github-pages-to-Cloudinary.md b/_posts/2019-11-20-How-to-move-images-in-Github-pages-to-Cloudinary.md new file mode 100644 index 0000000000000..7b39b7bf2f92d --- /dev/null +++ b/_posts/2019-11-20-How-to-move-images-in-Github-pages-to-Cloudinary.md @@ -0,0 +1,107 @@ +--- +layout: post +title: How to move images in Github pages to Cloudinary? +comment: true +description: Use Cloudinary to host images from your post to get relevant image format and right quality of image based on your device +image: /images/blog/dandelion-2817950_1280.9b587226.jpg +--- + +>tl;dr: Instead of hosting images for your blog on Github account, consider using the free tier of Cloudinary. It saves on usage at Github and allows you to generate some cool transformation and better format of images at the same time. Download the source code from [here](https://gist.github.com/akshayranganath/c387b3d011f6da096dbf82bac3a9039d). + +![change banner](/images/blog/dandelion-2817950_1280.jpg) + +## Current issue + +I have been publishing my blog on Github pages for quite some time now. I really like the approach and the super optimal HTML that is produced. However, when I run performance audits like [Page Speed Insights](https://developers.google.com/speed/pagespeed/insights/), I would get warnings about large images and for not using newer formats like `.webp`. There was no easy way to embed the different format of images without directly embedding `` tags. + +![Page speed insights report](/images/blog/psi_report.png) + + +I knew that Cloudinary would support the browser-specific format images. However, there was no direct way to migrate all the images. There was no easy way to search/replace the reference to the image as well. So I decided to build this functionality using simple python code. Here's how to do it. It's quite simple - we just need _Algorithms_ :-) + +![Algorithms](https://imgs.xkcd.com/comics/here_to_help_2x.png) +Source:[https://imgs.xkcd.com/comics/here_to_help_2x.png](https://imgs.xkcd.com/comics/here_to_help_2x.png) + +## Overall flow + +Here's the basic work-flow for the migration: + +* Look for all images by checking for the pattern `![]()`. +* If the image does not begin with a `http` or `https`, it will be migrated. Store the image path for next step. +* Using [Cloudinary's Upload Python SDK](https://cloudinary.com/documentation/django_integration), upload the image from local folder to Cloudinary +* Extract the URL of the uploaded image. Append the right set of transformations and replace the orginal image's relative path with the Cloudnary URL +* Save the blog posts and commit it back to Github. + +Let's look the work flow in details. You can download the full source code from [here](https://gist.github.com/akshayranganath/c387b3d011f6da096dbf82bac3a9039d). + +### Setting up dependencies + +To work with Cloudinary's Python SDK, we just need to install one library `cloudinary`. Just run this command. + +```text +pip install cloudinary +``` + +Next, [configure your Cloudinary credentials](https://cloudinary.com/documentation/django_integration#installation). I prefer to set the credentials in my environment variable but, let's use the direct method for simplicity. So this is how the beginning of your could would look. + +```python +import cloudinary +import cloudinary.uploader + +cloudinary.config( + cloud_name = '', + api_key = 'XXXXXXX', + api_secret = 'XXXXXXX' +) +``` + +### Build a list of images + +Since I am using Jekyll based static site generator, all my blog posts are in the folder `_posts`. So this task is simply opening each blog post and finding the images. Store this in a dictionary. Save the `alt` text as well. This will be useful to quickly rebuild the new tag. + +### Upload images + +Next, go through the list of images and upload it to Cloudinary. WHen an image is uploaded, it is referenced with a `public_id`. In this case, I am trying to re-use the same name as the local path. Here's the code to upload the images. + +```python +resp = cloudinary.uploader.upload( + object, + public_id=urllib.parse.quote(alt_text.strip()), + invlidate=True, + folder="blog" + ) +``` + +In this code, I am stating that: + +* public id is same as the local file path. This will ensure that the folder structure is retained. +* `invalidate=True` will ensure that any subsequent overwrites will cause the CDN cache to be invalidated. +* `folder=blog` will ensure that the root for all my images will be a folder named `blog`. + +The response URL will be of this format. The response JSON has a field called `secure_url` and it is this field we use to get back the uploaded URL. +[/images/blog/nut%20ball%20and%20bearing.jpg](/images/blog/nut%20ball%20and%20bearing.jpg) + +When publishing, I don't want to use the `v1568243786` bit. It represents the specific version of image variant. If we over-write it, the version may change. However, simply referencing it as this URL will work as well: +[/images/blog/nut%20ball%20and%20bearing.jpg](/images/blog/nut%20ball%20and%20bearing.jpg) + +So I run this cleanup and store the new URL. + +### Include Transformation + +For this simple migration, I am adding 2 transformations: + +* [`f_auto`](https://cloudinary.com/documentation/image_transformation_reference#format_parameter): This will ensure we will get back the right image format like a `.webp` on Chrome devices. +* [`q_auto`](https://cloudinary.com/documentation/image_transformation_reference#quality_parameter): This will set the best quality based on the image details. + +The combination wold translate to a transformation string of `f_quto,q_auto`. The resulting URL would now look as this: +[/images/blog/nut%20ball%20and%20bearing.jpg](/images/blog/nut%20ball%20and%20bearing.jpg) + +### Find & Replace + +The last step is to open each blog and find & replace the local image reference with Cloudinary URL. This is a simple string match again and a lookup into the dictionary. From the dictionary, we pull out the alt-text and Cloudinary URL and re-build the image markdown code. + +### Save and publish + +That's it. You can now commit the code and push it to Github. That should be create the updated website and images should be performant! Here's the Page Speed report after the change :-) + +![page speed report - after](/images/blog/psi_report_after.png) \ No newline at end of file diff --git a/_posts/2019-12-11-notes-from-instagram-engineering-performance-effort.md b/_posts/2019-12-11-notes-from-instagram-engineering-performance-effort.md new file mode 100644 index 0000000000000..05fb0a0c96f1b --- /dev/null +++ b/_posts/2019-12-11-notes-from-instagram-engineering-performance-effort.md @@ -0,0 +1,65 @@ +--- +layout: post +title: Notes from Making Instagram Fast articles +comment: true +description: Instagram Engineering published 4 part blog on web performance optimization. Here is my notes from these articles. +image: /images/blog/lmpszilmigl2dkkmbure +--- + +## &tl;dr; +> [Instagram Engineering](https://instagram-engineering.com/) published a series of 4-part blog series on how they improved the performance on their website. Here's my notes from these articles for a quick overview. + +![child racing on a go-kart](/images/blog/lmpszilmigl2dkkmbure) + + +## Part 1 + +[Making Instagram.com faster: Part 1](https://instagram-engineering.com/making-instagram-com-faster-part-1-62cc0c327538) + +* intelligent pre-loading of fonts and GraphQL queries + * _Approach 1_: (suggested by [Chrome team](https://medium.com/reloading/a-link-rel-preload-analysis-from-the-chrome-data-saver-team-5edf54b08715))Have `preload` tag after the critical ` + + + + + diff --git a/resume.md b/resume.md new file mode 100644 index 0000000000000..ee28bdc7ef686 --- /dev/null +++ b/resume.md @@ -0,0 +1,40 @@ +--- +layout: page +title: Résumé +permalink: /resume/ +--- + +## Akshay Ranganath +[akshay@akshayranganath@com](mailto:akshay@akshayranganath@com) +[LinkedIn Profile](https://www.linkedin.com/in/akshayranganath) +[Twitter](https://twitter.com/rakshay) +[Github](https://github.com/akshayranganath) + + + + + +
\ No newline at end of file diff --git a/sitemap.md b/sitemap.md new file mode 100644 index 0000000000000..5c92e0c29b2de --- /dev/null +++ b/sitemap.md @@ -0,0 +1,48 @@ +--- +layout: page +title: Sitemap +permalink: /sitemap/ +description: Collated list of my blogs on topics related to DevOps core concepts, DevOps tools, Web Performance, front-end performance optimization and Search Engine Optimization +--- + +## Development & DevOps Tools +- [Basic Git Commands](https://akshayranganath.github.io/Basic-Git-Commands/) +- [DevOps Journey: Setting up a Git Server and Client](https://akshayranganath.github.io/DevOps-Journey-Setting-up-Git-server-and-Client) +- [Handling a Git Pull request with merge conflict](https://akshayranganath.github.io/Git-Pull-Handling-Merge-Conflict/) +- [Performance Testing with Siege and Jenkins](https://akshayranganath.github.io/Performance-Testing-with-Jenkins/) +- [Basic Docker Commands](https://akshayranganath.github.io/Docker-Commands/) +- [Integraring Akamai with Varnish - The Varnish Connector training notes](https://akshayranganath.github.io/Akamai-Varnish-Connector-Training/) +- [Creating virtual environments with for Python](https://akshayranganath.github.io/How-to-create-dev-environment-for-python/) +- [JSON Pretty Printer for command line](https://akshayranganath.github.io/pretty-print-json-with-command-line/) +- [OCSP Validation with OpenSSL](https://akshayranganath.github.io/OCSP-Validation-With-Openssl/) +- [Using python pandas for log analysis]() +- [Python stats from Jenkins job output](https://akshayranganath.github.io/Python-Stats-From-Jenkins-Job-Output/) + + + +## DevOps Core Concepts +- [How to learn & adapt devops?](https://akshayranganath.github.io/how-to-learn-and-adapt-to-devops/) +- [Quotes from the book Phoenix Project](https://akshayranganath.github.io/quotes-from-book-the-phoenix-project/) +- [AWS Summit SFO - Day 1 notes](https://akshayranganath.github.io/AWSSummit-SFO-Day1-Notes/) +- [AWS Summit SFO - Keynote & Day 2 notes](https://akshayranganath.github.io/AWSSummit-SFO-Day2-Keynote-Notes/) +- [Markdown for notes](https://akshayranganath.github.io/Markdown-for-notes/) +- [Quotes from DevOps Handbook](https://akshayranganath.github.io/quotes-from-devops-handbook/) +- [Notes from DevOps Enterprise Summit, 2017](https://akshayranganath.github.io/Notes-From-Enterprise-DevOps-Summit-San-Francisco/) + + +## Webperformance & Search Engine Optimization +- [Useful Search Engine Optimization Tags - Part 1](https://akshayranganath.github.io/Useful-SEO-tags-part-1/) +- [Useful Search Engine Optimization Tags - Part 2](https://akshayranganath.github.io/Useful-SEO-Tags-Part-2/) +- [What metrics matter for SEO?](https://akshayranganath.github.io/what-metrics-matter-for-seo/) +- [How can a Content Delivery Network help in your SEO?](https://akshayranganath.github.io/how-can-cdn-help-in-your-seo-efforts/) +- [Web Performance - Notes from PerfPlanet December calendar blogs](https://akshayranganath.github.io/webperformance-notes-from-perfplanet/) +- [Content Security Policy & How to avoid mixed content?](https://akshayranganath.github.io/csp-to-avoid-mixed-content/) +- [Mutual Auth and Certificate Revocationn](https://akshayranganath.github.io/mutual-auth-certificate-revocation/) +- [Migrating Wordpress blogs to Github pages](https://akshayranganath.github.io/Migrating-Wordpress-Blogs-to-Github-Pages/) +- [Perceived Performance study & Perceptual SpeedIndex](https://akshayranganath.github.io/Perceived-Performance-Study-SpeedPerception/) +- [Wordpress vs Github pages - Performance compared](https://akshayranganath.github.io/Wordpress-vs-Github-Pages/) +- [5 DIY Tools for SEO Analysis](https://akshayranganath.github.io/5-Free-Tools-For-DIY-SEO-Analysis/) +- [Synthetic testing with Jenkins and Wordpress](https://akshayranganath.github.io/Synthetic-Testing-With-WPT-and-Jenkins/) +- [3 Reasons why AMP & CDNs can work together](https://akshayranganath.github.io/3-Reasons-Why-AMP-And-CDN-Can-Work-Together/) +- [3 Trends in Web Performance](https://akshayranganath.github.io/3-Trends-in-Web-Performance/) +- [Quick Primer on SPA and SPA tracking with RUM](https://akshayranganath.github.io/Understand_SPA_and_Tracking_SPA_pages_with_RUM/) diff --git a/style.scss b/style.scss index 3915a90244691..f0b02d09fb29a 100644 --- a/style.scss +++ b/style.scss @@ -25,7 +25,7 @@ body { .container { margin: 0 auto; - max-width: 740px; + max-width: 1024px; padding: 0 10px; width: 100%; } @@ -108,6 +108,7 @@ strong, b { img { max-width: 100%; + height: auto; } // Fixes images in popup boxes from Google Translate diff --git a/tags.html b/tags.html new file mode 100644 index 0000000000000..469ec672378ee --- /dev/null +++ b/tags.html @@ -0,0 +1,28 @@ +--- +layout: page +title: Tags +--- + +
+
+ {% for tag in site.tags %} + {{ tag[0] }} + {% endfor %} +
+
+
+ {% for tag in site.tags %} +

{{ tag[0] }}

+ + {% endfor %} +
+