package main import ( "bufio" "fmt" "log" "net/url" "os" "strings" "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" "github.com/otiai10/copy" ) const memory = ":memory:" var db *sqlite3.Conn func initDb() error { var err error var sqlstmt string // Not really necessary because I am currently saving the db in memory sqlstmt = ` DROP TABLE IF EXISTS artists; DROP TABLE IF EXISTS websites; DROP TABLE IF EXISTS links; ` err = db.Exec(sqlstmt) if err != nil { log.Fatal(err) } sqlstmt = ` CREATE TABLE IF NOT EXISTS artists ( id integer NOT NULL CONSTRAINT artists_pk PRIMARY KEY AUTOINCREMENT, name text NOT NULL ); CREATE TABLE IF NOT EXISTS websites ( id integer NOT NULL CONSTRAINT websites_pk PRIMARY KEY AUTOINCREMENT, name text NOT NULL ); CREATE TABLE IF NOT EXISTS links ( id integer NOT NULL CONSTRAINT links_pk PRIMARY KEY AUTOINCREMENT, url text NOT NULL, websites_id integer NOT NULL, artists_id integer NOT NULL, CONSTRAINT links_artists FOREIGN KEY (artists_id) REFERENCES artists (id), CONSTRAINT links_websites FOREIGN KEY (websites_id) REFERENCES websites (id) ); ` err = db.Exec(sqlstmt) if err != nil { return err } sqlstmt = ` SELECT name FROM sqlite_schema WHERE type ='table' AND name NOT LIKE 'sqlite_%'; ` stmt, _, err := db.Prepare(sqlstmt) if err != nil { return err } defer stmt.Close() for stmt.Step() { log.Println("created table:", stmt.ColumnText(0)) } if err := stmt.Err(); err != nil { return err } err = stmt.Close() if err != nil { return err } return nil } func getArtistFromUrl(Url string) string { if strings.Contains(Url, "rule34.xxx") { unescapedUrl, err := url.QueryUnescape(Url) if err != nil { log.Fatal(err) } return unescapedUrl[51:] } if strings.Contains(Url, "kemono.su") || strings.Contains(Url, "coomer.su") { return strings.Split(Url, "/")[5] } return "" } func trimUrl(url string) string { // we can look for specific strings like encoded chars and then remove them disallowedStrings := []string{ "%20", "%21", "%22", "%23", "%24", "%26", "%27", "%28", "%29", "%2A", "%2C", "%2E", "%2F", "%3B", "%3C", "%3E", "%3F", "%5B", "%5C", "%5D", "%5E", "%60", "%7B", "%7C", "%7D", "%7E", "artist", } for _, disallowedString := range disallowedStrings { url = strings.ReplaceAll(url, disallowedString, "") } return strings.TrimRight(url, "-_") } func convert(fileLine string) error { log.Println("starting conversion on:", fileLine) oldPath := fmt.Sprintf("Artists/%s", getArtistFromUrl(fileLine)) fileLine = trimUrl(fileLine) log.Println("trimmed into:", fileLine) if strings.Contains(fileLine, "%") { return fmt.Errorf("%s", "URL contains Encoded Unicode and a direct conversion cannot be reasonably made") } artist := getArtistFromUrl(fileLine) if strings.Compare(artist, "") == 0 { return fmt.Errorf("%s", "Trying to get the artist name from the url failed") } log.Println("artist name:", artist) newPath := fmt.Sprintf("Artists/%s", artist) if len(artist) <= 1 { return fmt.Errorf("%s", "Trying to remove unwanted stuff from artist name has caused it to become too small") } if strings.Compare(oldPath, newPath) == 0 { return nil } log.Printf("oldPath: %s\tnewPath: %s\n", oldPath, newPath) err := copy.Copy(oldPath, newPath) if err != nil { return err } os.Rename(oldPath, fmt.Sprint("Artist/__", artist)) return nil } func parseLinks(filename string) error { file, err := os.Open(filename) if err != nil { return err } scanner := bufio.NewScanner(file) for scanner.Scan() { var line string line = scanner.Text() if strings.HasPrefix(line, "#") { continue } if !strings.Contains(line, "#") { err = convert(line) if err != nil { log.Printf("convert: %s\n", err) } fmt.Println() continue } /* split_url := strings.Split(line, "#") url := strings.TrimSpace(split_url[0]) artist := strings.TrimSpace(split_url[1]) log.Printf("url: %s\tartist: %s\n", url, artist) */ } err = file.Close() if err != nil { return err } return nil } func main() { var err error db, err = sqlite3.Open(memory) if err != nil { log.Fatal(err) } err = initDb() if err != nil { log.Fatal(err) } err = parseLinks("gallery-dl/Rule34.links") if err != nil { log.Fatal(err) } err = db.Close() if err != nil { log.Fatal(err) } }