Golang网站文件扫描工具

作者: Luan 分类: 编程开发 发布时间: 2018-01-25 15:07

以前学习Go,做的一个扫描网站文件的小工具。

package main

import (
	"flag"
	"strings"
	"os"
	"bufio"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"
	"io"
	"io/ioutil"
)

var urls chan string
var wg sync.WaitGroup
var zip_codes map[string]string

func getTime()(_ string){
	return "[" + time.Now().Format("2006-01-02 15:04:05")[11:] + "] "
}

func worker(method string, count_int int, timeOut int, keepAlive bool){
	client := &http.Client{
		Timeout: time.Duration(timeOut) * time.Second,
        Transport: http.DefaultTransport,
    }

	count := strconv.Itoa(count_int);
	for len(urls) > 0{
		url := <- urls;

		data := url_scan(client,method,url,timeOut,keepAlive);
		if data != nil{
			if data["status"] != "404" {
				fmt.Print("\033[K" + getTime() + url + "\t[" + data["status"] + "] [" + data["size"] + "] [" + data["type"] + "]\n");
			}
		}
		cur := strconv.Itoa(count_int - len(urls))
		for len(cur) != len(count){
			cur = "0" + cur;
		}
		fmt.Print("\033[K" + getTime() + "[" + cur + "/" + count + "] " + url + "\r");
	}
	wg.Done()
}

func url_scan(client *http.Client, method string, url string, timeOut int, keepAlive bool)(data map[string]string){
	// 访问URL
	req, _ := http.NewRequest(method, url, nil)
    req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:51.0) Gecko/20100101 Firefox/51.0")
    if keepAlive{
    	req.Header.Set("Connection", "keep-alive")
    }else{
    	req.Header.Set("Connection", "close")
    }
    resp, err := client.Do(req)
    if err != nil {
    	return nil
    }
    
    // 提取数据
    data = make(map[string]string);
    data["status"] = strconv.Itoa(resp.StatusCode);
    if len(resp.Header["Content-Length"]) != 0{
    	data["size"] = resp.Header["Content-Length"][0];
    }else{
    	data["size"] = "0";
   	}
	data["type"] = "unknow";
	response := bufio.NewReader(resp.Body);
	for zip_code_name, zip_code_value := range zip_codes{
		code,_ := response.Peek(len(zip_code_value));
		if string(code) == zip_code_value{
			data["type"] = zip_code_name;
		}
	}
	io.Copy(ioutil.Discard,resp.Body)
    resp.Body.Close();
    return data
}

func main(){
	// 输出Banner
	fmt.Println("BkScan Code By Luan");

	// 获取命令行参数
	method := flag.String("m","GET","HTTP Method");
	urls_name := flag.String("f","","Target Urls");
	url := flag.String("u","","Target Url");
	dic_name := flag.String("dic","dic.txt","Dictionary");
	exts_name := flag.String("ext","","File extension");
	threadCount := flag.Int("threads",2,"Thread Count");
	timeOut := flag.Int("timeout",2,"Request Time Out");
	keepAlive := flag.Bool("keep-alive",true,"Http Keep-alive");
	flag.Parse();

	if *url == "" && *urls_name == ""{
		fmt.Println("BkScan -h for help");
		os.Exit(0);
	}
	fmt.Println(urls_name);

	zip_codes = map[string]string{
		"zip": "PK",
		"7z": "7z",
		"gzip": "\x1f\x8B\x08",
		"bz2": "\x42\x5A\x68",
		"rar": "Rar",
	}

	// 生成后缀格式列表
	zip_exts := [...] string {"tar","gz","tar.gz","zip","rar","biz","7z","swp","log"}
	var exts []string;
	if *exts_name == "Zip"{
		exts = zip_exts[:];
	}else{
		for _, ext := range strings.Split(*exts_name,","){
			exts = append(exts,ext);
		}
	}

	// 根据字典和后缀列表生成待访问的列表
	f , err := os.Open(*dic_name);
	if err != nil{
		panic(err);
	}
	var urlList []string;
	allURLTxt := bufio.NewScanner(f);
	for allURLTxt.Scan() {
		for i := 0;i < len(exts);i ++{
			newurl := *url + allURLTxt.Text()
			if exts[i] != ""{
				newurl += "." + exts[i];
			}
			urlList = append(urlList, newurl);
		}
	}

	// 把列表放到信道里面去
	urls = make(chan string, len(urlList))
	for _, url := range urlList {
		urls <- url
	}
	count_int := len(urls);

	// 开始
	fmt.Println(getTime() + "BkScan Start");
	for i := 0; i < *threadCount; i++ {
		wg.Add(1);
		go worker(*method,count_int,*timeOut,*keepAlive);
	}
	wg.Wait();

	fmt.Println("\n" + getTime() + "End");
}

 

当时遇到的主要的问题是,如何保持HTTP长连接和不读取整个HTTP响应内容。

参考的文章:

如何关闭Golang中的HTTP连接 How to Close Golang’s HTTP connection

http://www.cnblogs.com/cobbliu/p/4517598.html

golang的http client源码简析

http://studygolang.com/articles/5774

 

另外调试的时候遇到过一个小坑,Burpsuite的默认配置是强制把keep-alive改成close。

如果觉得我的文章对您有用,请随意打赏。您的支持将鼓励我继续创作!

发表评论