Download images from old Ghost blog

ghosttohugo/card.go

@@ -3,6 +3,8 @@ package ghosttohugo
 import (
 	"bytes"
 	"fmt"
+	"regexp"
+	"strings"
 
 	jww "github.com/spf13/jwalterweatherman"
 )
@@ -197,28 +199,40 @@ func cardImage(payload interface{}) string {
 		return ""
 	}
 
+	img := src.(string)
+	if fileName, err := ImgDownloader.Download(src.(string)); err == nil {
+		img = fileName
+	}
 	if caption, ok := m["caption"]; ok {
 		return fmt.Sprintf(
 			"{{< figure src=\"%s\" caption=\"%s\" >}}\n",
-			stripContentFolder(src.(string)),
+			stripContentFolder(img),
 			caption,
 		)
 	}
-
 	return fmt.Sprintf(
 		"{{< figure src=\"%s\" >}}\n",
-		stripContentFolder(src.(string)),
+		stripContentFolder(img),
 	)
 }
 
+var re = regexp.MustCompile(`(?m)__GHOST_URL__.+?(\))`)
+
 func cardMarkdown(payload interface{}) string {
 	m, ok := payload.(map[string]interface{})
 	if !ok {
 		jww.ERROR.Println("cardMarkdown: payload not correct type")
 		return ""
 	}
+
 	if markdown, ok := m["markdown"]; ok {
-		return fmt.Sprintf("%s\n", markdown.(string))
+		markdownStr := markdown.(string)
+		for _, match := range re.FindAllString(markdownStr, -1) {
+			if fileName, err := ImgDownloader.Download(strings.TrimSuffix(match, ")")); err == nil {
+				markdownStr = strings.ReplaceAll(markdownStr, match, fileName+")")
+			}
+		}
+		return fmt.Sprintf("%s\n", markdownStr)
 	}
 	return ""
 }

ghosttohugo/card_test.go

@@ -135,6 +135,8 @@ func Test_cardImage(t *testing.T) {
 			"{{< figure src=\"test\" caption=\"caption\" >}}\n",
 		},
 	}
+
+	NewImgDownloader("", "", false)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			if got := cardImage(tt.args.payload); got != tt.want {

ghosttohugo/converter.go

@@ -3,6 +3,7 @@ package ghosttohugo
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"strings"
@@ -128,6 +129,21 @@ func (c Converter) populatePost(p *post) {
 			}
 		}
 	}
+
+	if fileName, err := c.downloadImage(p.Image); err == nil {
+		p.Image = fileName
+	}
+	if fileName, err := c.downloadImage(p.FeaturedImage); err == nil {
+		p.FeaturedImage = fileName
+	}
+
+}
+
+func (c Converter) downloadImage(img string) (string, error) {
+	if len(img) <= 0 {
+		return "", fmt.Errorf("no url surpplied")
+	}
+	return ImgDownloader.Download(img)
 }
 
 // Convert is the main function of this package. It takes an io.ReadSeeker

ghosttohugo/img_downloader.go

@@ -0,0 +1,78 @@
+package ghosttohugo
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+type imageDownloader struct {
+	siteUrl  string
+	basePath string
+	insecure bool
+}
+
+var ImgDownloader *imageDownloader
+
+func NewImgDownloader(path string, url string, inscure bool) {
+	ImgDownloader = &imageDownloader{
+		siteUrl:  url,
+		basePath: path,
+		insecure: inscure,
+	}
+}
+
+func (imgD imageDownloader) Download(img string) (string, error) {
+
+	if imgD.siteUrl == "" {
+		return "", fmt.Errorf("no base url defined. unable to download use --url to provide one")
+	}
+
+	URL := strings.Replace(img, "__GHOST_URL__", imgD.siteUrl, -1)
+	imageName := filepath.Base(img)
+	jww.DEBUG.Printf("Downloading image: %s", imageName)
+
+	if filepath.Ext(imageName) == "" || len(filepath.Ext(imageName)) > 5 {
+		imageName = imageName + ".jpg"
+	}
+
+	fileName := path.Join("images", filepath.Base(imageName))
+	savePath := path.Join(imgD.basePath, "static", fileName)
+
+	transCfg := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: imgD.insecure}, // ignore expired SSL certificates
+	}
+	client := &http.Client{Transport: transCfg}
+
+	response, err := client.Get(URL)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != 200 {
+		return "", errors.New("received non 200 response code")
+	}
+	//Create a empty file
+	file, err := os.Create(savePath)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	//Write the bytes to the fiel
+	_, err = io.Copy(file, response.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return "/" + fileName, nil
+}

ghosttohugo/post.go

@@ -3,6 +3,7 @@ package ghosttohugo
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"path/filepath"
 	"strings"
 	"time"
@@ -76,14 +77,18 @@ func (p post) frontMatter() map[string]interface{} {
 	return metadata
 }
 
+func (p *post) fileName() string {
+	return fmt.Sprintf("%s-%s.md", p.Published.Format("2006-01-02"), p.Slug)
+}
+
 func (c *Converter) writePost(p post) error {
 	jww.DEBUG.Printf("converting: %s", p.Title)
 	path := filepath.Join(c.path, "content")
 	switch p.isPage() {
 	case true:
-		path = filepath.Join(path, p.Slug+".md")
+		path = filepath.Join(path, p.fileName())
 	case false:
-		path = filepath.Join(path, "post", p.Slug+".md")
+		path = filepath.Join(path, "post", p.fileName())
 	}
 
 	buf := bytes.NewBuffer(nil)
@@ -95,14 +100,14 @@ func (c *Converter) writePost(p post) error {
 	}
 
 	switch {
-	case p.Content != "":
-		if _, err := buf.Write([]byte(p.Content)); err != nil {
-			return err
-		}
 	case p.MobileDoc != "":
 		if _, err := buf.Write([]byte(p.mobiledocMarkdown())); err != nil {
 			return err
 		}
+	case p.Content != "":
+		if _, err := buf.Write([]byte(p.Content)); err != nil {
+			return err
+		}
 	default:
 		if _, err := buf.Write([]byte(p.Plain)); err != nil {
 			return err

ghosttohugo/site.go

@@ -42,6 +42,7 @@ func (c *Converter) createSite() error {
 	mkdir(c.path, "content")
 	mkdir(c.path, "archetypes")
 	mkdir(c.path, "static")
+	mkdir(c.path, "static/images")
 	mkdir(c.path, "data")
 	mkdir(c.path, "themes")
 

main.go

@@ -20,14 +20,16 @@ func usage() {
 func main() {
 
 	var (
-		path, loc, format     string
-		force, verbose, debug bool
+		path, loc, format, url          string
+		force, verbose, debug, insecure bool
 	)
 
 	flag.Usage = usage
 
 	flag.StringVarP(&path, "hugo", "p", "newhugosite",
 		"path to create the new hugo project")
+	flag.StringVarP(&url, "url", "u", "",
+		"base url of ghost blog to download images from example http://blog.example.com (default: \"\")")
 	flag.StringVarP(&loc, "location", "l", "",
 		"location to use for time conversions (default: local)")
 	flag.StringVarP(&format, "dateformat", "d", "2006-01-02 15:04:05",
@@ -38,6 +40,8 @@ func main() {
 		"print verbose logging output")
 	flag.BoolVarP(&debug, "debug", "", false,
 		"print verbose logging output")
+	flag.BoolVarP(&insecure, "insecure", "", false,
+		"do not check SSL cert of download content")
 
 	flag.Parse()
 
@@ -66,6 +70,8 @@ func main() {
 	}
 
 	c, err := ghosttohugo.New(opts...)
+	ghosttohugo.NewImgDownloader(path, url, insecure)
+
 	if err != nil {
 		jww.FATAL.Fatalf("Error initializing converter (%v)\n", err)
 	}