Skip to content

Commit b81eb0c

Browse files
mye956mye956
committed
Adding unit tests for fault request ordering
1 parent ec1598e commit b81eb0c

File tree

1 file changed

+268
-3
lines changed

1 file changed

+268
-3
lines changed

‎ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go‎

Lines changed: 268 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ const (
5858
tcLatencyFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","parent":"1:1","options":{"limit":1000,"delay":{"delay":123456789,"jitter":4567,"correlation":0},"ecn":false,"gap":0}}]`
5959
tcLossFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","dev":"eth0","parent":"1:1","options":{"limit":1000,"loss-random":{"loss":0.06,"correlation":0},"ecn":false,"gap":0}}]`
6060
tcCommandEmptyOutput = `[]`
61+
startEndpoint = "/api/%s/fault/v1/%s/start"
62+
stopEndpoint = "/api/%s/fault/v1/%s/stop"
6163
)
6264

6365
var (
@@ -1899,9 +1901,13 @@ func generateStopNetworkPacketLossTestCases() []networkFaultInjectionTestCase {
18991901
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
19001902
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
19011903
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
1902-
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel)
1903-
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD)
1904-
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil)
1904+
gomock.InOrder(
1905+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel),
1906+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD),
1907+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil),
1908+
)
1909+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(2).Return(mockCMD)
1910+
mockCMD.EXPECT().CombinedOutput().Times(2).Return([]byte(""), nil)
19051911
},
19061912
},
19071913
{
@@ -2047,3 +2053,262 @@ func TestCheckNetworkPacketLoss(t *testing.T) {
20472053
tcs := generateCheckNetworkPacketLossTestCases()
20482054
testNetworkFaultInjectionCommon(t, tcs, NetworkFaultPath(types.PacketLossFaultType, types.CheckNetworkFaultPostfix))
20492055
}
2056+
2057+
func TestNetworkFaultRequestOrdering(t *testing.T) {
2058+
tcs := []struct {
2059+
name string
2060+
faultType string
2061+
requestBody interface{}
2062+
setAgentStateExpectations func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient)
2063+
setExecExpectations func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller)
2064+
}{
2065+
{
2066+
name: types.BlackHolePortFaultType + "request ordering",
2067+
faultType: types.BlackHolePortFaultType,
2068+
requestBody: happyBlackHolePortReqBody,
2069+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2070+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2071+
Return(happyTaskResponse, nil).
2072+
Times(2)
2073+
},
2074+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
2075+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2076+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2077+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2078+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2079+
// We can enforce the ordering of exec mock calls.
2080+
gomock.InOrder(
2081+
// Exec mocks for start black hole port request
2082+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2083+
// Sleep for 2 seconds to mock that the request is taking some time
2084+
time.Sleep(2 * time.Second)
2085+
}).Times(1).Return(startCtx, startCancel),
2086+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2087+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(iptablesChainNotFoundError), errors.New("exit status 1")),
2088+
exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, true),
2089+
exec.EXPECT().GetExitCode(gomock.Any()).Times(1).Return(1),
2090+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2091+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2092+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2093+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2094+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2095+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2096+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2097+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2098+
2099+
// Exec mocks for stop black hole port request
2100+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2101+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2102+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2103+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2104+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2105+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2106+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2107+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2108+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
2109+
)
2110+
},
2111+
},
2112+
{
2113+
name: types.LatencyFaultType + "request ordering",
2114+
faultType: types.LatencyFaultType,
2115+
requestBody: happyNetworkLatencyReqBody,
2116+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2117+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2118+
Return(happyTaskResponse, nil).
2119+
Times(2)
2120+
},
2121+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
2122+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2123+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2124+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2125+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2126+
// We can enforce the ordering of exec mock calls.
2127+
gomock.InOrder(
2128+
// Exec mocks for start latency request
2129+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2130+
// Sleep for 2 seconds to mock that the request is taking some time
2131+
time.Sleep(2 * time.Second)
2132+
}).Times(1).Return(startCtx, startCancel),
2133+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2134+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2135+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2136+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2137+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2138+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2139+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2140+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2141+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2142+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2143+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2144+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2145+
2146+
// Exec mocks for stop latency request
2147+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2148+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2149+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultExistsCommandOutput), nil),
2150+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2151+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2152+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2153+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2154+
)
2155+
},
2156+
},
2157+
{
2158+
name: types.PacketLossFaultType + "request ordering",
2159+
faultType: types.PacketLossFaultType,
2160+
requestBody: happyNetworkPacketLossReqBody,
2161+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2162+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).
2163+
Return(happyTaskResponse, nil).
2164+
Times(2)
2165+
},
2166+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
2167+
startCtx, startCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2168+
stopCtx, stopCancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2169+
cmdExec := mock_execwrapper.NewMockCmd(ctrl)
2170+
// We want to ensure that the start fault request executes and finishes first before the stop fault request.
2171+
// We can enforce the ordering of exec mock calls.
2172+
gomock.InOrder(
2173+
// Exec mocks for start packet loss request
2174+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Do(func(_, _ interface{}) {
2175+
// Sleep for 2 seconds to mock that the request is taking some time
2176+
time.Sleep(2 * time.Second)
2177+
}).Times(1).Return(startCtx, startCancel),
2178+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2179+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2180+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2181+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2182+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2183+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2184+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2185+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2186+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2187+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2188+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2189+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2190+
2191+
// Exec mocks for stop packet loss request
2192+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(stopCtx, stopCancel),
2193+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2194+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLossFaultExistsCommandOutput), nil),
2195+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2196+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2197+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec),
2198+
cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(""), nil),
2199+
)
2200+
},
2201+
},
2202+
}
2203+
2204+
for _, tc := range tcs {
2205+
t.Run(tc.name, func(t *testing.T) {
2206+
// Mocks
2207+
ctrl := gomock.NewController(t)
2208+
defer ctrl.Finish()
2209+
2210+
agentState := mock_state.NewMockAgentState(ctrl)
2211+
metricsFactory := mock_metrics.NewMockEntryFactory(ctrl)
2212+
2213+
router := mux.NewRouter()
2214+
mockExec := mock_execwrapper.NewMockExec(ctrl)
2215+
handler := New(agentState, metricsFactory, mockExec)
2216+
networkConfigClient := netconfig.NewNetworkConfigClient()
2217+
2218+
tc.setAgentStateExpectations(agentState, networkConfigClient)
2219+
tc.setExecExpectations(mockExec, ctrl)
2220+
2221+
var startHandleMethod, stopHandleMethod func(http.ResponseWriter, *http.Request)
2222+
switch tc.faultType {
2223+
case types.BlackHolePortFaultType:
2224+
startHandleMethod = handler.StartNetworkBlackholePort()
2225+
stopHandleMethod = handler.StopNetworkBlackHolePort()
2226+
case types.LatencyFaultType:
2227+
startHandleMethod = handler.StartNetworkLatency()
2228+
stopHandleMethod = handler.StopNetworkLatency()
2229+
case types.PacketLossFaultType:
2230+
startHandleMethod = handler.StartNetworkPacketLoss()
2231+
stopHandleMethod = handler.StopNetworkPacketLoss()
2232+
default:
2233+
t.Error("Unrecognized network fault type")
2234+
}
2235+
router.HandleFunc(
2236+
NetworkFaultPath(tc.faultType, types.StartNetworkFaultPostfix),
2237+
startHandleMethod,
2238+
).Methods(http.MethodPost)
2239+
2240+
router.HandleFunc(
2241+
NetworkFaultPath(tc.faultType, types.StopNetworkFaultPostfix),
2242+
stopHandleMethod,
2243+
).Methods(http.MethodPost)
2244+
2245+
var requestBody io.Reader
2246+
reqBodyBytes, err := json.Marshal(tc.requestBody)
2247+
require.NoError(t, err)
2248+
requestBody = bytes.NewReader(reqBodyBytes)
2249+
startReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(startEndpoint, endpointId, tc.faultType), requestBody)
2250+
require.NoError(t, err)
2251+
2252+
ch1 := make(chan struct {
2253+
int
2254+
error
2255+
})
2256+
2257+
reqBodyBytes, err = json.Marshal(tc.requestBody)
2258+
require.NoError(t, err)
2259+
requestBody = bytes.NewReader(reqBodyBytes)
2260+
stopReq, err := http.NewRequest(http.MethodPost, fmt.Sprintf(stopEndpoint, endpointId, tc.faultType), requestBody)
2261+
require.NoError(t, err)
2262+
2263+
ch2 := make(chan struct {
2264+
int
2265+
error
2266+
})
2267+
2268+
// Make an asynchronous Start request first
2269+
go makeAsyncRequest(router, startReq, ch1)
2270+
2271+
// Waiting a bit before sending the stop request
2272+
time.Sleep(1 * time.Second)
2273+
2274+
// Make an asynchronous Stop request second
2275+
go makeAsyncRequest(router, stopReq, ch2)
2276+
2277+
// Waiting to get the status code of the start request
2278+
resp1 := <-ch1
2279+
require.NoError(t, resp1.error)
2280+
assert.Equal(t, http.StatusOK, resp1.int)
2281+
2282+
// Waiting to get the status code of the stop request
2283+
resp2 := <-ch2
2284+
require.NoError(t, resp2.error)
2285+
assert.Equal(t, http.StatusOK, resp2.int)
2286+
})
2287+
}
2288+
}
2289+
2290+
// Helper function for making asynchronous mock HTTP requests
2291+
func makeAsyncRequest(router *mux.Router, req *http.Request, ch chan<- struct {
2292+
int
2293+
error
2294+
}) {
2295+
defer close(ch)
2296+
2297+
// Makes a mock HTTP request
2298+
recorder := httptest.NewRecorder()
2299+
router.ServeHTTP(recorder, req)
2300+
2301+
var actualResponseBody types.NetworkFaultInjectionResponse
2302+
err := json.Unmarshal(recorder.Body.Bytes(), &actualResponseBody)
2303+
if err != nil {
2304+
ch <- struct {
2305+
int
2306+
error
2307+
}{-1, err}
2308+
} else {
2309+
ch <- struct {
2310+
int
2311+
error
2312+
}{recorder.Code, nil}
2313+
}
2314+
}

0 commit comments

Comments
 (0)