imagecrawler/imagecrawler.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/user"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Global vars
var wg sync.WaitGroup

type picture struct {
	FileURL     string `json:"file_url"`
	Width       int    `json:"width"`
	Height      int    `json:"height"`
	ImageWidth  int    `json:"image_width"`
	ImageHeight int    `json:"image_height"`
	FileExt     string `json:"file_ext"`
	Tags        string `json:"tag_string"`
}

// main function to download pictures
func main() {

	// define flags and parse them
	var path string
	var safemode bool
	var tags string
	var aspect string
	var site string

	// variables for downloading
	picHits := 1
	page := 1
	ratio := 0.0

	flag.StringVar(&path, "dir", "unnamed", "Directory to safe pictures. Default is %HOME/pictures/konachan/unnamed")
	flag.BoolVar(&safemode, "safe", false, "Safemode to filter NSFW pictures. Default is false")
	flag.StringVar(&tags, "tags", "", "Tags used to filter search query.")
	flag.StringVar(&aspect, "aspect", "", "Aspect ratio pics should have")
	flag.StringVar(&site, "site", "konachan", "Site to crawl from, either konachan or danbooru")

	flag.Parse()

	// set home directory and create it to save pictures in
	homepath, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	filepath := strings.Join([]string{homepath.HomeDir, "pictures", site, strings.TrimSuffix(path, "\n")}, "/")
	os.MkdirAll(filepath, 0700)

	// edit tags array to met API requirement
	tags = strings.Replace(tags, ",", "+", -1)
	tags = strings.Replace(tags, "=", ":", -1)
	tags = strings.TrimSuffix(tags, "\n")

	// calculate aspect ratio
	if isFlagPassed("aspect") {
		aspectSlice := strings.Split(aspect, ":")
		widthF, _ := strconv.ParseFloat(aspectSlice[0], 64)
		heightF, _ := strconv.ParseFloat(aspectSlice[1], 64)
		ratio = widthF / heightF
	} else {
		ratio = 0.0
	}

	for picHits > 0 {

		fmt.Println("Page: ", page)

		website := fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s", page, tags)
		if safemode {
			website = fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s+rating:safe", page, tags)
		}

		if site == "danbooru" {
			website = fmt.Sprintf("https://danbooru.donmai.us/posts.json?page=%d&tags=%s", page, tags)
			if safemode {
				website = fmt.Sprintf("https://danbooru.donmai.us/posts.json?page=%d&tags=%s+rating:safe", page, tags)
			}
		}

		picList := openConnection(website)
		pictures, count := parseMaps(picList, ratio)

		picHits = count
		page++

		wg.Add(len(pictures))
		for _, pic := range pictures {
			go downloadPic(pic, filepath)
		}
		wg.Wait()
		time.Sleep(1 * time.Second)
	}
}

func isFlagPassed(name string) bool {
	found := false
	flag.Visit(func(f *flag.Flag) {
		if f.Name == name {
			found = true
		}
	})
	return found
}

// function to create the connection to konachan and get the API response
func openConnection(url string) []picture {
	var f []picture

	result, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer result.Body.Close()

	data, err := io.ReadAll(result.Body)
	if err != nil {
		log.Fatal(err)
	}

	if err = json.Unmarshal(data, &f); err != nil {
		panic(err)
	}

	return f
}

// function to parse the json response and extract only the file url
func parseMaps(f []picture, ratio float64) ([]string, int) {
	fileURLs := []string{}
	picCount := 0

	if isFlagPassed("aspect") {
		for _, pic := range f {
			picCount++

			picWidthF := 1.0
			picHeightF := 1.0

			if pic.Width != 0 && pic.Height != 0 {
				picWidthF = float64(pic.Width)
				picHeightF = float64(pic.Height)
			} else {
				picWidthF = float64(pic.ImageWidth)
				picHeightF = float64(pic.ImageHeight)
			}

			if (picWidthF / picHeightF) == ratio {
				fileURL := pic.FileURL
				fileURLs = append(fileURLs, fileURL)
			}
		}
	} else {
		for _, pic := range f {
			picCount++
			fileURL := pic.FileURL
			fileURLs = append(fileURLs, fileURL)
		}
	}
	return fileURLs, picCount
}

// function to download and sace the pictures to disk
func downloadPic(picURL string, filepath string) {
	defer wg.Done()

	picName, err := url.PathUnescape(strings.Split(picURL, "/")[len(strings.Split(picURL, "/"))-1])
	if err != nil {
		log.Fatal(err)
	}

	if _, err := os.Stat(filepath + "/" + picName); err == nil {
		return
	}

	result, err := http.Get(picURL)
	if err != nil {
		log.Fatal(err)
	}
	defer result.Body.Close()

	//fmt.Println(result.Status)

	if result.StatusCode != 200 {
		wg.Add(1)
		go downloadPic(picURL, filepath)
		return
	}

	file, err := os.Create(filepath + "/" + picName)
	if err != nil {
		log.Fatal(err)
	}

	_, err = io.Copy(file, result.Body)
	if err != nil {
		log.Fatal(err)
	}

	file.Close()

	fmt.Printf("Downloading: %s\n", picName)
}