日常开发中，我们除了使用charles、finder抓包外，也可以使用mitmproxy抓包，并且它还提供二次开发能力。在学习mitmproxy之前，我们先学习下他的go版本的精简实现github.com/lqqyt2423/go-mitmproxy/cmd/go-mitmproxy
目前只支持http代理和https代理，不支持websocket和透明代理，这是比mitmproxy弱的地方，它提供了11个hook点，我们可以根据需求，通过addon的方式来定义和添加自己的hook点实现自定义抓包，它的11个hook点定义如下：
代码语言：javascript复制type Addon interface {
// 一个客户端已经连接到了mitmproxy。请注意，一个连接可能对应多个HTTP请求。
ClientConnected(*ClientConn)
// 一个客户端连接已关闭（由我们或客户端关闭）。
ClientDisconnected(*ClientConn)
// mitmproxy 已连接到服务器。
ServerConnected(*ConnContext)
// 服务器连接已关闭（由我们或服务器关闭）。
ServerDisconnected(*ConnContext)
// 与服务器的TLS握手已成功完成。
TlsEstablishedServer(*ConnContext)
// HTTP请求头已成功读取。此时，请求体为空。
Requestheaders(*Flow)
// 完整的HTTP请求已被读取。
Request(*Flow)
// HTTP响应头已成功读取。此时，响应体为空。
Responseheaders(*Flow)
// 完整的HTTP响应已被读取。
Response(*Flow)
// 流式请求体修改器
StreamRequestModifier(*Flow, io.Reader) io.Reader
// 流式响应体修改器
StreamResponseModifier(*Flow, io.Reader) io.Reader
}下面我们开始分析下它的源码，入口函数位于cmd/go-mitmproxy/main.go，里面定义了Config结构体，每一个field代表了一个抓包工具的可选项：
代码语言：javascript复制type Config struct {
version bool // show go-mitmproxy version
Addr string // proxy listen addr
WebAddr string // web interface listen addr
SslInsecure bool // not verify upstream server SSL/TLS certificates.
IgnoreHosts []string // a list of ignore hosts
AllowHosts []string // a list of allow hosts
CertPath string // path of generate cert files
Debug int // debug mode: 1 - print debug log, 2 - show debug from
Dump string // dump filename
DumpLevel int // dump level: 0 - header, 1 - header body
Upstream string // upstream proxy
UpstreamCert bool // Connect to upstream server to look up certificate details. Default: True
MapRemote string // map remote config filename
MapLocal string // map local config filename
filename string // read config from the filename
}它的main函数里核心包括了下面四步：
1，初始化proxy
代码语言：javascript复制p, err := proxy.NewProxy(opts)代理结构体的定义位于proxy/proxy.go
代码语言：javascript复制type Proxy struct {
Opts *Options
Version string
Addons []Addon
entry *entry
attacker *attacker
shouldIntercept func(req *http.Request) bool // req is received by proxy.server
upstreamProxy func(req *http.Request) (*url.URL, error) // req is received by proxy.server, not client request
}初始化proxy的时候，会设置对应的entry和attacker
2，设置本地和远程拦截策略，也就是根据规则匹配对应的host，并做出是否代理的决策
代码语言：javascript复制p.SetShouldInterceptRule(func(req *http.Request) bool {
return !helper.MatchHost(req.Host, config.IgnoreHosts)
})
p.SetShouldInterceptRule(func(req *http.Request) bool {
return helper.MatchHost(req.Host, config.AllowHosts)
})在proxy上绑定这个拦截规则
代码语言：javascript复制func (proxy *Proxy) SetShouldInterceptRule(rule func(req *http.Request) bool) {
proxy.shouldIntercept = rule
}mapremote和maplocal的具体实现如下：
addon/mapremote.go
代码语言：javascript复制if err := helper.NewStructFromFile(filename, &mapRemote); err != nil {
if err := mapRemote.validate(); err != nil {
type MapRemote struct {
proxy.BaseAddon
Items []*mapRemoteItem
Enable bool
}addon/maplocal.go
代码语言：javascript复制if err := helper.NewStructFromFile(filename, &mapLocal); err != nil {
if err := mapLocal.validate(); err != nil {type MapLocal struct {
proxy.BaseAddon
Items []*mapLocalItem
Enable bool
}3，添加hook点，也就是addon
代码语言：javascript复制p.AddAddon(proxy.NewUpstreamCertAddon(false))
p.AddAddon(&proxy.LogAddon{})
p.AddAddon(web.NewWebAddon(config.WebAddr))
mapRemote, err := addon.NewMapRemoteFromFile(config.MapRemote)
p.AddAddon(mapRemote)
mapLocal, err := addon.NewMapLocalFromFile(config.MapLocal)
p.AddAddon(mapLocal)
dumper := addon.NewDumperWithFilename(config.Dump, config.DumpLevel)
p.AddAddon(dumper)添加hook点就是把拦截点append到数组里就行了
代码语言：javascript复制func (proxy *Proxy) AddAddon(addon Addon) {
proxy.Addons = append(proxy.Addons, addon)
}proxy/addon.go中对hook点进行了约定，也就是说用户可以在这11个hook点上随意定义hook插件
代码语言：javascript复制type Addon interface {
// A client has connected to mitmproxy. Note that a connection can correspond to multiple HTTP requests.
ClientConnected(*ClientConn)
// A client connection has been closed (either by us or the client).
ClientDisconnected(*ClientConn)
// Mitmproxy has connected to a server.
ServerConnected(*ConnContext)
// A server connection has been closed (either by us or the server).
ServerDisconnected(*ConnContext)
// The TLS handshake with the server has been completed successfully.
TlsEstablishedServer(*ConnContext)
// HTTP request headers were successfully read. At this point, the body is empty.
Requestheaders(*Flow)
// The full HTTP request has been read.
Request(*Flow)
// HTTP response headers were successfully read. At this point, the body is empty.
Responseheaders(*Flow)
// The full HTTP response has been read.
Response(*Flow)
// Stream request body modifier
StreamRequestModifier(*Flow, io.Reader) io.Reader
// Stream response body modifier
StreamResponseModifier(*Flow, io.Reader) io.Reader
// onAccessProxyServer
AccessProxyServer(req *http.Request, res http.ResponseWriter)
}同时也定义了一个基础插件和两个最常用的插件实现
代码语言：javascript复制type BaseAddon struct{}type LogAddon struct {
BaseAddon
}type UpstreamCertAddon struct {
BaseAddon
UpstreamCert bool // Connect to upstream server to look up certificate details.
}func NewUpstreamCertAddon(upstreamCert bool) *UpstreamCertAddon {
return &UpstreamCertAddon{UpstreamCert: upstreamCert}
}一些其他的addon定义如下：
web/web.go里定义了webAddOn
代码语言：javascript复制type WebAddon struct {
proxy.BaseAddon
server *http.Server
upgrader *websocket.Upgrader
conns []*concurrentConn
connsMu sync.RWMutex
flowMessageState map[*proxy.Flow]messageType
flowMu sync.Mutex
}实现了一个简单的httpserver，通过websocket将抓包结果发送到web前端，我们可以通过网页查看抓包结果
代码语言：javascript复制web := &WebAddon{
flowMessageState: make(map[*proxy.Flow]messageType),
}
web.upgrader = &websocket.Upgrader{
serverMux := new(http.ServeMux)
serverMux.HandleFunc("/echo", web.echo)
fsys, err := fs.Sub(assets, "client/build")
serverMux.Handle("/", http.FileServer(http.FS(fsys)))
web.server = &http.Server{Addr: addr, Handler: serverMux}
web.conns = make([]*concurrentConn, 0)
err := web.server.ListenAndServe()addon/dumper.go 定义了Dumper将抓包结果保存到文件里
代码语言：javascript复制out, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
return NewDumper(out, level)type Dumper struct {
proxy.BaseAddon
out io.Writer
level int // 0: header 1: header body
}4，启动代理
代码语言：javascript复制log.Fatal(p.Start())启动proxy的时候分别会启动两个协程，一个是entry，一个是attacker
代码语言：javascript复制func (proxy *Proxy) Start() error {
go func() {
if err := proxy.attacker.start(); err != nil {
return proxy.entry.start()4.1首先看下entry协程
它的定义位于 proxy/entry.go，它代表了代理server，本质上就是一个http/https服务器
代码语言：javascript复制type entry struct {
proxy *Proxy
server *http.Server
}func newEntry(proxy *Proxy) *entry {
e := &entry{proxy: proxy}
e.server = &http.Server{func (e *entry) start() error {
ln, err := net.Listen("tcp", addr)
pln := &wrapListener{
Listener: ln,
proxy: e.proxy,
}
return e.server.Serve(pln)重点看下ServeHTTP函数，每当有拦截到来的时候先处理hook，然后初始化attacker的连接，并调用对应的attack方法
代码语言：javascript复制func (e *entry) ServeHTTP(res http.ResponseWriter, req *http.Request) {
if req.Method == "CONNECT" {
e.handleConnect(res, req)
// trigger addon event Responseheaders
for _, addon := range e.proxy.Addons {
addon.Responseheaders(f)
}
for _, addon := range proxy.Addons {
addon.AccessProxyServer(req, res)
}
proxy.attacker.initHttpDialFn(req)
proxy.attacker.attack(res, req)处理连接的时候，会进行请求的拷贝，也就是entry收到的请求到accacker的请求，以及attacker响应到entry的响应
代码语言：javascript复制func (e *entry) handleConnect(res http.ResponseWriter, req *http.Request) {
e.directTransfer(res, req, f)
e.httpsDialFirstAttack(res, req, f)
e.httpsDialLazyAttack(res, req, f)具体拷贝方法定义位于
proxy/helper.go
代码语言：javascript复制func transfer(log *log.Entry, server, client io.ReadWriteCloser) {
go func() {
_, err := io.Copy(server, client)
go func() {
_, err := io.Copy(client, server)拷贝完请求之后就需要向真是上游发起请求
代码语言：javascript复制func (e *entry) httpsDialLazyAttack(res http.ResponseWriter, req *http.Request, f *Flow) {
cconn, err := e.establishConnection(res, f)
peek, err := cconn.(*wrapClientConn).Peek(3)
conn, err := proxy.attacker.httpsDial(req.Context(), req)
transfer(log, conn, cconn)func (e *entry) httpsDialFirstAttack(res http.ResponseWriter, req *http.Request, f *Flow) {
conn, err := proxy.attacker.httpsDial(req.Context(), req)
cconn, err := e.establishConnection(res, f)
peek, err := cconn.(*wrapClientConn).Peek(3)
if !helper.IsTls(peek) {
// todo: http, ws
transfer(log, conn, cconn)
// is tls
f.ConnContext.ClientConn.Tls = true
proxy.attacker.httpsTlsDial(req.Context(), cconn, conn)
每一次代理请求被定义为一个flow，其代码实现位于：proxy/flow.go
代码语言：javascript复制func newFlow() *Flow {
return &Flow{Id: uuid.NewV4(),
done: make(chan struct{}),}
}4.2 接下来看下attacker的协程
其代码位于proxy/attacker.go ，创建attacker的时候需要创建一个ca也就是一个自签名的证书，方便被代理的服务使用（代理https时候）
代码语言：javascript复制ca, err := newCa(proxy.Opts)
a := &attacker{
proxy: proxy,
ca: ca,
client: &http.Client{
Transport: &http.Transport
a.server = &http.Server{
Handler: a,
ConnContext:
a.h2Server = &http2.Server{
MaxConcurrentStreams: 100, // todo: wait for remote server setting
NewWriteScheduler: func() http2.WriteScheduler { return http2.NewPriorityWriteScheduler(nil) },
}func newCa(opts *Options) (cert.CA, error) {
newCaFunc := opts.NewCaFunc
if newCaFunc != nil {
return newCaFunc()
}
return cert.NewSelfSignCA(opts.CaRootPath)
}在它的start方法里启动了代理服务
代码语言：javascript复制func (a *attacker) start() error {
return a.server.Serve(a.listener)
}代理连接是在初始化函数里定义的
代码语言：javascript复制func (a *attacker) initHttpDialFn(req *http.Request) {
connCtx.dialFn = func(ctx context.Context) error {
addr := helper.CanonicalAddr(req.URL)
c, err := a.proxy.getUpstreamConn(ctx, req)
if err := a.serverTlsHandshake(ctx, connCtx); err != nil {
serverConn := newServerConn()
serverConn.Conn = cw
serverConn.Address = addr
serverConn.client = &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return cw, nil
},attack方法是代理逻辑的核心
代码语言：javascript复制func (a *attacker) attack(res http.ResponseWriter, req *http.Request) {
reply := func(response *Response, body io.Reader) {
res.Header().Add(key, v)
_, err := io.Copy(res, body)
_, err := io.Copy(res, response.BodyReader)
_, err := res.Write(response.Body)
proxyReq, err := http.NewRequestWithContext(proxyReqCtx, f.Request.Method, f.Request.URL.String(), reqBody)
proxyReq.Header.Add(key, v)
proxyRes, err = a.client.Do(proxyReq)
if err := f.ConnContext.dialFn(req.Context()); err != nil {
proxyRes, err = f.ConnContext.ServerConn.client.Do(proxyReq)
reply(f.Response, resBody)对于https代理，需要发起tls连接
代码语言：javascript复制func (a *attacker) httpsTlsDial(ctx context.Context, cconn net.Conn, conn net.Conn) {
clientTlsConn := tls.Server(cconn, &tls.Config{
SessionTicketsDisabled: true, // 设置此值为 true ，确保每次都会调用下面的 GetConfigForClient 方法
GetConfigForClient: func(chi *tls.ClientHelloInfo) (*tls.Config, error) {
c, err := a.ca.GetCert(chi.ServerName)
go func() {
if err := clientTlsConn.HandshakeContext(ctx); err != nil {
if err := a.serverTlsHandshake(ctx, connCtx); err != nil {
// will go to attacker.ServeHTTP
a.serveConn(clientTlsConn, connCtx)
func newCa(opts *Options) (cert.CA, error) {
newCaFunc := opts.NewCaFunc
if newCaFunc != nil {
return newCaFunc()
}
return cert.NewSelfSignCA(opts.CaRootPath)
}
// send clientHello to server, server handshakefunc (a *attacker) serverTlsHandshake(ctx context.Context, connCtx *ConnContext) error {
serverTlsConfig := &tls.Config{
serverTlsConn := tls.Client(serverConn.Conn, serverTlsConfig)
if err := serverTlsConn.HandshakeContext(ctx); err != nil {
serverTlsState := serverTlsConn.ConnectionState()
serverConn.tlsState = &serverTlsState
serverConn.client = &http.Client{
Transport: &http.Transport{自认证证书逻辑位于cert/self_sign_ca.go，创建证书的逻辑如下
代码语言：javascript复制storePath, err := getStorePath(path)
ca := &SelfSignCA{
StorePath: storePath,
cache: lru.New(100),
group: new(singleflight.Group),
}
if err := ca.load(); err != nil {
key, err := x509.ParsePKCS8PrivateKey(keyDERBlock.Bytes)
x509Cert, err := x509.ParseCertificate(certDERBlock.Bytes)
if err := ca.create(); err != nil {证书的名字
代码语言：javascript复制func (ca *SelfSignCA) caFile() string {
return filepath.Join(ca.StorePath, "mitmproxy-ca.pem")
}func (ca *SelfSignCA) create() error {
key, cert, err := createCert()
key, err := rsa.GenerateKey(rand.Reader, 2048)
template := &x509.Certificate{
certBytes, err := x509.CreateCertificate
cert, err := x509.ParseCertificate
return ca.saveCert()
