remove comments, replace fdw table to native table

Author: ermolaev

CHANGELOG.md

@@ -1,5 +1,38 @@
 # Changelog
 
+## 0.2.6    2018-03-11
+
+* Fix regular expressions to support schema qualification changes in 10.3
+
+
+## 0.2.5    2017-11-15
+
+* Filter out indices belonging partitioned tables
+
+
+## 0.2.4    2017-11-02
+
+* Remove pg_buffercache extension if present (its only used for statistics purposes)
+* Remove extension comments if present - they can prevent non-superusers from
+  restoring the tables, and are never used together with Rails anyway
+
+
+## 0.2.3    2017-10-21
+
+* pg 10.x adds AS Integer to structure.sql format [Nathan Woodhull](https://github.com/woodhull)
+
+
+## 0.2.2    2017-08-05
+
+* Support Rails 5.1 primary key UUIDs that rely on gen_random_uuid()
+
+
+## 0.2.1    2017-06-30
+
+* Allow primary keys to be the last column of a table [Clemens Kofler](https://github.com/clemens)
+  - Special thanks to [Jon Mohrbacher](https://github.com/johnnymo87) who submitted a similar earlier change
+
+
 ## 0.2.0    2017-03-20
 
 * Reduce dependencies to only require ActiveRecord [Mario Uher](https://github.com/ream88)

Gemfile.lock

@@ -1,36 +1,39 @@
 PATH
   remote: .
   specs:
-    activerecord-clean-db-structure (0.2.1)
+    activerecord-clean-db-structure (0.2.6)
       activerecord (>= 4.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.1.2)
-      activesupport (= 5.1.2)
-    activerecord (5.1.2)
-      activemodel (= 5.1.2)
-      activesupport (= 5.1.2)
+    activemodel (5.1.5)
+      activesupport (= 5.1.5)
+    activerecord (5.1.5)
+      activemodel (= 5.1.5)
+      activesupport (= 5.1.5)
       arel (~> 8.0)
-    activesupport (5.1.2)
+    activesupport (5.1.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (8.0.0)
     concurrent-ruby (1.0.5)
-    i18n (0.8.4)
-    minitest (5.10.2)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    minitest (5.11.3)
+    rake (0.9.6)
     thread_safe (0.3.6)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   activerecord-clean-db-structure!
+  rake (~> 0)
 
 BUNDLED WITH
-   1.15.1
+   1.16.1

activerecord-clean-db-structure.gemspec

@@ -16,4 +16,6 @@ Gem::Specification.new do |s|
   s.license       = 'MIT'
 
   s.add_dependency('activerecord', '>= 4.2')
+
+  s.add_development_dependency 'rake', '~> 0'
 end

lib/activerecord-clean-db-structure/clean_dump.rb

@@ -18,40 +18,61 @@ def run
 
       # Remove pg_stat_statements extension (its not relevant to the code)
       dump.gsub!(/^CREATE EXTENSION IF NOT EXISTS pg_stat_statements.*/, '')
-      dump.gsub!(/^COMMENT ON EXTENSION pg_stat_statements.*/, '')
       dump.gsub!(/^-- Name: (EXTENSION )?pg_stat_statements;.*/, '')
 
+      # Remove pg_buffercache extension (its not relevant to the code)
+      dump.gsub!(/^CREATE EXTENSION IF NOT EXISTS pg_buffercache.*/, '')
+      dump.gsub!(/^-- Name: (EXTENSION )?pg_buffercache;.*/, '')
+
+      # Remove comments on extensions, they create problems if the extension is owned by another user
+      dump.gsub!(/^COMMENT ON EXTENSION .*/, '')
+
       # Remove useless, version-specific parts of comments
       dump.gsub!(/^-- (.*); Schema: (public|-); Owner: -.*/, '-- \1')
 
       # Remove useless comment lines
       dump.gsub!(/^--$/, '')
 
-      # Mask user mapping
-      dump.gsub!(/^CREATE USER MAPPING FOR \w+ SERVER (\w+) .*?;/m, 'CREATE USER MAPPING FOR some_user SERVER \1;')
-      dump.gsub!(/^-- Name: USER MAPPING \w+ SERVER (\w+); Type: USER MAPPING/, '-- Name: USER MAPPING some_user SERVER \1; Type: USER MAPPING')
+      # remove comments
+      dump.gsub!(/^-- Name.+ Type: (COMMENT|EXTENSION|INDEX|TABLE|TYPE)/, '')
+
+      ### NEED FOR TEST DATABASE `rails db:setup`
+        # remove server and user for FDW
+        dump.gsub!(/^-- Name: [\w ]+; Type: SERVER.+?;/m, '')
+        dump.gsub!(/^-- Name: [\w ]+; Type: USER MAPPING.+?;/m, '')
+
+        # FDW table to native table
+        dump.gsub!(/^CREATE FOREIGN TABLE (.+?)\nSERVER.+?;/m, "CREATE TABLE \\1;")
+      ###
 
       # Reduce noise for id fields by making them SERIAL instead of integer+sequence stuff
       #
       # This is a bit optimistic, but works as long as you don't have an id field thats not a sequence/uuid
-      dump.gsub!(/^CREATE TABLE (\w+) \(\n    id integer NOT NULL(.*?);/m, "CREATE TABLE \\1 (\n    id SERIAL PRIMARY KEY\\2;")
-      dump.gsub!(/^CREATE TABLE (\w+) \(\n    id bigint NOT NULL(.*?);/m, "CREATE TABLE \\1 (\n    id BIGSERIAL PRIMARY KEY\\2;")
-
-      dump.gsub!(/^    id uuid DEFAULT uuid_generate_v4\(\) NOT NULL,$/, '    id uuid DEFAULT uuid_generate_v4() PRIMARY KEY,')
-      dump.gsub!(/^CREATE SEQUENCE \w+_id_seq\s+START WITH 1\s+INCREMENT BY 1\s+NO MINVALUE\s+NO MAXVALUE\s+CACHE 1;$/, '')
-      dump.gsub!(/^ALTER SEQUENCE \w+_id_seq OWNED BY .*;$/, '')
-      dump.gsub!(/^ALTER TABLE ONLY \w+ ALTER COLUMN id SET DEFAULT nextval\('\w+_id_seq'::regclass\);$/, '')
-      dump.gsub!(/^ALTER TABLE ONLY \w+\s+ADD CONSTRAINT \w+_pkey PRIMARY KEY \(id\);$/, '')
+      dump.gsub!(/^CREATE TABLE ([\w.]+) \(\n    id integer NOT NULL(.*?);/m, "CREATE TABLE \\1 (\n    id SERIAL PRIMARY KEY\\2;")
+      dump.gsub!(/^CREATE TABLE ([\w.]+) \(\n    id bigint NOT NULL(.*?);/m, "CREATE TABLE \\1 (\n    id BIGSERIAL PRIMARY KEY\\2;")
+
+      dump.gsub!(/^    id uuid DEFAULT (public\.)?uuid_generate_v4\(\) NOT NULL(,)?$/, '    id uuid DEFAULT \1uuid_generate_v4() PRIMARY KEY\2')
+      dump.gsub!(/^    id uuid DEFAULT (public\.)?gen_random_uuid\(\) NOT NULL(,)?$/, '    id uuid DEFAULT \1gen_random_uuid() PRIMARY KEY\2')
+      dump.gsub!(/^CREATE SEQUENCE [\w\.]+_id_seq\s+(AS integer\s+)?START WITH 1\s+INCREMENT BY 1\s+NO MINVALUE\s+NO MAXVALUE\s+CACHE 1;$/, '')
+      dump.gsub!(/^ALTER SEQUENCE [\w\.]+_id_seq OWNED BY .*;$/, '')
+      dump.gsub!(/^ALTER TABLE ONLY [\w\.]+ ALTER COLUMN id SET DEFAULT nextval\('[\w\.]+_id_seq'::regclass\);$/, '')
+      dump.gsub!(/^ALTER TABLE ONLY [\w\.]+\s+ADD CONSTRAINT [\w\.]+_pkey PRIMARY KEY \(id\);$/, '')
       dump.gsub!(/^-- Name: (\w+\s+)?id; Type: DEFAULT$/, '')
       dump.gsub!(/^-- .*_id_seq; Type: SEQUENCE.*/, '')
       dump.gsub!(/^-- Name: (\w+\s+)?\w+_pkey; Type: CONSTRAINT$/, '')
 
       # Remove inherited tables
-      inherited_tables_regexp = /-- Name: ([\w_]+); Type: TABLE\n\n[^;]+?INHERITS \([\w_]+\);/m
+      inherited_tables_regexp = /-- Name: ([\w_\.]+); Type: TABLE\n\n[^;]+?INHERITS \([\w_\.]+\);/m
       inherited_tables = dump.scan(inherited_tables_regexp).map(&:first)
       dump.gsub!(inherited_tables_regexp, '')
       inherited_tables.each do |inherited_table|
-        dump.gsub!(/ALTER TABLE ONLY #{inherited_table}[^;]+;/, '')
+        dump.gsub!(/ALTER TABLE ONLY (public\.)?#{inherited_table}[^;]+;/, '')
+
+        index_regexp = /CREATE INDEX ([\w_]+) ON (public\.)?#{inherited_table}[^;]+;/m
+        dump.scan(index_regexp).map(&:first).each do |inherited_table_index|
+          dump.gsub!("-- Name: #{inherited_table_index}; Type: INDEX", '')
+        end
+        dump.gsub!(index_regexp, '')
       end
 
       # Remove whitespace between schema migration INSERTS to make editing easier

lib/activerecord-clean-db-structure/version.rb

@@ -1,3 +1,3 @@
 module ActiveRecordCleanDbStructure
-  VERSION = '0.2.1'
+  VERSION = '0.2.6'
 end