一个在线服务要上线,常会出现,测试机正常放到线上就出错的情况。
我们团队在上周就经历过这种情况。
解决的办法一般有log重放和旁路测试。相对来说,旁路测试更有效。
所谓旁路测试是制将 对线上服务器LS访问的流量,拷贝一份给测试服务程序TS。而响应返回则是有LS完成,TS的返回被丢弃或者记录
我写了一个通用的程序(基于tiwstd,理解思路不基于也很容易,基于twisted 能够很快完成一个可用的系统,当然也可以自己封装网络通信框架实现灵活和快捷,这是题外话),能够支持旁路测试的无缝切换,且能够支持一组旁路测试服务器。(也即旁路测试服务器可以有TS1 TS2...) 。
程序如下(我现在就在旁路测试情况下发文的,nklog系统目前就在旁路测试中:)),今天太晚了就不解释了。
from twisted.internet.protocol import Protocol, ClientFactory, ServerFactoryfrom twisted.internet import reactor import time import sys#ABTestServer A is Online Server ;B is Testing Server#The IPtable rules to support follows:#/sbin/iptables -t nat -A PREROUTING -p tcp -i eth0 -s ! 60.28.222.137 -d 60.28.222.137 --dport 8301 -j DNAT --to 60.28.222.137:80#echo 1 >/proc/sys/net/ipv4/ip_forward class ServerHost: host="" port=0class ServerConnector: passclass ForwardServer(Protocol): def __init__(self, hostA, portA,hostsB): self.hostA = hostA self.portA = portA self.hostsB = hostsB self.data = "" self._connected = False def dataReceived(self, data): #From Client To FWServer self.data += data if (len(self.data) > 0): self.connectorA.connector.transport.write(self.data) for aBConnector in self.connectorsB: aBConnector.connector.transport.write(self.data) self.data = "" def connectionMade(self): self.beginTime=time.time() print "INFO:: client connection made" aClientFC = ForwardClientFactory(self,1) aServConnector=ServerConnector() aServConnector.connector = reactor.connectTCP(self.hostA, self.portA, aClientFC) self.connectorA=aServConnector self.connectorsB=[] for aBHost in self.hostsB: bClientFC = ForwardClientFactory(self,0) bServConnector=ServerConnector() bServConnector.connector = reactor.connectTCP(aBHost.host, aBHost.port, bClientFC) self.connectorsB.append( bServConnector ) def setConnected(self, flag): if flag: self.onConnected() else: self.transport.loseConnection() self._connected = flag def onConnected(self): if len(self.data) > 0: self.connectorA.connector.transport.write(self.data) for aBConnector in self.connectorsB: aBConnector.connector.transport.write(self.data) self.data = "" def connectionLost(self, reason): self.connectorA.connector.transport.loseConnection() for aBConnector in self.connectorsB: aBConnector.connector.transport.loseConnection() self._connected = False print "Lost Connection "+ str( (time.time()-self.beginTime) )+" secs"class ForwardClient(Protocol): def __init__(self, forward,ctype): self.forward = forward self.type=ctype
self.bConnected=False def dataReceived(self, data): #From FWServer to Client# print self.type if self.type !=0 : #Send To A only self.forward.transport.write(data) def connectionMade(self): self.bConnected=True def connectionLost(self, reason): self.bConnected=False if self.type!=0: self.forward.transport.loseConnection() self.forward.connectionLost("client lost" else: pass #self.transport.loseConnection()class ForwardServerFactory(ServerFactory): def __init__(self, host, port,hostsB): self.host = host self.port = port self.hostsB=hostsB def buildProtocol(self, addr): return ForwardServer(self.host, self.port,hostsB)class ForwardClientFactory(ClientFactory): def __init__(self, forward,ctype): self.forward = forward self.type=ctype def buildProtocol(self, addr): return ForwardClient(self.forward,self.type) def clientConnectionFailed(self, connector, reason): self.forward.transport.loseConnection() self.forward.connectionLost("client connect failed"if __name__ == "__main__": if len(sys.argv) != 5: print "USAGE: %s " % sys.argv[0] sys.exit(1) host, port, listen_port = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) strFHostsB=sys.argv[4] fHostsB=open(strFHostsB) lines = fHostsB.readlines() hostsB=[] for aLine in lines: aLine=aLine.replace("
","" aLine=aLine.replace("
","" segs = aLine.split(" " if len(segs) != 2: continue servHost = ServerHost() servHost.host=segs[0] servHost.port=int(segs[1]) hostsB.append(servHost) print "INFO:: Load B Host "+str(len(hostsB)) server_factory = ForwardServerFactory(host, port,hostsB) reactor.listenTCP(listen_port, server_factory) reactor.run()
举例说明如下
http服务旁路测试数据转发
80--iptables->8300(python abtest.py ...)
8300--abtester-->80(apache )
8300--abtester-->8310(tcpeserver 0.0.0.0 8310./a.out)