一个在线服务要上线,常会出现,测试机正常放到线上就出错的情况。
我们团队在上周就经历过这种情况。
解决的办法一般有log重放和旁路测试。相对来说,旁路测试更有效。
所谓旁路测试是制将 对线上服务器LS访问的流量,拷贝一份给测试服务程序TS。而响应返回则是有LS完成,TS的返回被丢弃或者记录
我写了一个通用的程序(基于tiwstd,理解思路不基于也很容易,基于twisted 能够很快完成一个可用的系统,当然也可以自己封装网络通信框架实现灵活和快捷,这是题外话),能够支持旁路测试的无缝切换,且能够支持一组旁路测试服务器。(也即旁路测试服务器可以有TS1 TS2...) 。
程序如下(我现在就在旁路测试情况下发文的,nklog系统目前就在旁路测试中:)),今天太晚了就不解释了。
from twisted.internet.protocol import Protocol, ClientFactory, ServerFactoryfrom twisted.internet import reactor  import time   import sys#ABTestServer A is Online Server ;B is Testing Server#The IPtable rules to support follows:#/sbin/iptables -t nat -A PREROUTING -p tcp -i eth0 -s ! 60.28.222.137 -d 60.28.222.137   --dport 8301  -j DNAT --to 60.28.222.137:80#echo 1 >/proc/sys/net/ipv4/ip_forward class ServerHost:    host=""    port=0class ServerConnector:        passclass ForwardServer(Protocol):    def __init__(self, hostA, portA,hostsB):        self.hostA = hostA        self.portA = portA        self.hostsB = hostsB        self.data = ""        self._connected = False    def dataReceived(self, data): #From Client To FWServer         self.data += data        if (len(self.data) > 0):            self.connectorA.connector.transport.write(self.data)        for aBConnector in self.connectorsB:            aBConnector.connector.transport.write(self.data)            self.data = ""    def connectionMade(self):        self.beginTime=time.time()        print "INFO:: client connection made"        aClientFC = ForwardClientFactory(self,1)        aServConnector=ServerConnector()        aServConnector.connector = reactor.connectTCP(self.hostA, self.portA, aClientFC)        self.connectorA=aServConnector        self.connectorsB=[]        for aBHost in self.hostsB:                bClientFC = ForwardClientFactory(self,0)                bServConnector=ServerConnector()                bServConnector.connector = reactor.connectTCP(aBHost.host, aBHost.port, bClientFC)                self.connectorsB.append( bServConnector )    def setConnected(self, flag):        if flag:            self.onConnected()        else:            self.transport.loseConnection()                                                                self._connected = flag      def onConnected(self):        if len(self.data) > 0:            self.connectorA.connector.transport.write(self.data)            for aBConnector in self.connectorsB:                aBConnector.connector.transport.write(self.data)            self.data = ""    def connectionLost(self, reason):        self.connectorA.connector.transport.loseConnection()        for aBConnector in self.connectorsB:            aBConnector.connector.transport.loseConnection()        self._connected = False        print "Lost Connection "+ str( (time.time()-self.beginTime) )+" secs"class ForwardClient(Protocol):    def __init__(self, forward,ctype):        self.forward = forward        self.type=ctype
self.bConnected=False    def dataReceived(self, data):  #From FWServer to Client#       print self.type        if self.type !=0 : #Send To A only                 self.forward.transport.write(data)    def connectionMade(self):        self.bConnected=True    def connectionLost(self, reason):        self.bConnected=False        if self.type!=0:            self.forward.transport.loseConnection()            self.forward.connectionLost("client lost"        else:            pass            #self.transport.loseConnection()class ForwardServerFactory(ServerFactory):    def __init__(self, host, port,hostsB):        self.host = host        self.port = port        self.hostsB=hostsB    def buildProtocol(self, addr):        return ForwardServer(self.host, self.port,hostsB)class ForwardClientFactory(ClientFactory):    def __init__(self, forward,ctype):        self.forward = forward        self.type=ctype    def buildProtocol(self, addr):        return ForwardClient(self.forward,self.type)    def clientConnectionFailed(self, connector, reason):        self.forward.transport.loseConnection()        self.forward.connectionLost("client connect failed"if __name__ == "__main__":    if len(sys.argv) != 5:        print "USAGE: %s    " % sys.argv[0]        sys.exit(1)    host, port, listen_port = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])                                                                                         strFHostsB=sys.argv[4]    fHostsB=open(strFHostsB)    lines = fHostsB.readlines()    hostsB=[]    for aLine in lines:        aLine=aLine.replace("
",""        aLine=aLine.replace("
",""        segs = aLine.split("	"        if len(segs) != 2:                continue        servHost = ServerHost()        servHost.host=segs[0]        servHost.port=int(segs[1])        hostsB.append(servHost)    print "INFO:: Load B Host "+str(len(hostsB))    server_factory = ForwardServerFactory(host, port,hostsB)    reactor.listenTCP(listen_port, server_factory)    reactor.run()
举例说明如下
http服务旁路测试数据转发
80--iptables->8300(python abtest.py ...)
8300--abtester-->80(apache )
8300--abtester-->8310(tcpeserver 0.0.0.0 8310./a.out)