[GO] Scrapping(1) - URL checker

샘플 사이트를 접속하여 상태 정보 및 응답 코드를 가져오는 샘플코드임.

- 고루틴, 자료구조는 map{string, slice} 구조를 사용함, key:url, value:status, errCode

// URL checker
package main

import (
	"fmt"
	"net/http"
	"strconv"
)

type requestResult struct {
	url     string
	status  string
	errCode int
}

func main() {
	ch := make(chan requestResult)
	// var results = make(map[string]string) // value를 string 으로 처리한경우
	var results = make(map[string][]string) // value를 slice 로 처리한경우

	urls := []string{
		"https://academy.nomadcoders.co/",
		"https://www.airbnb.com/",
		"https://www.google.com/",
		"https://www.amazon.com/",
		"https://www.reddit.com/",
		"https://soundcloud.com/",
		"https://www.facebook.com/",
		"https://www.instagram.com/",
	}

	for _, url := range urls {
		go hitURL(url, ch)
	}

	for i := 0; i < len(urls); i++ {
		result := <-ch
		// results[result.url] = result.status // value를 string으로 처리한경우
		results[result.url] = append(results[result.url], result.status, strconv.Itoa(result.errCode))
	}

	for url, status := range results {
		// fmt.Println(url, status) // value를 string으로 처리한경우
		fmt.Println(url, status[0], status[1])
	}
}

func hitURL(url string, ch chan<- requestResult) {
	resp, err := http.Get(url)
	status := "OK"
	errCode := 0

	if err != nil || resp.StatusCode >= 400 {
		status = "FAILED"
		errCode = resp.StatusCode
	} else {
		status = "OK"
		errCode = 0
	}
	ch <- requestResult{url: url, status: status, errCode: errCode}
}

 

[Running] go run "d:\workspace\GO\nomad\main.go"
https://www.instagram.com/ FAILED 429
https://www.facebook.com/ OK 0
https://www.amazon.com/ OK 0
https://academy.nomadcoders.co/ OK 0
https://www.reddit.com/ OK 0
https://www.airbnb.com/ OK 0
https://www.google.com/ OK 0
https://soundcloud.com/ OK 0

[Done] exited with code=0 in 1.889 seconds

 

참고자료 [https://unsungit.tistory.com/79?category=1052839]