H3qWi7pn

1#include <chrono>
2#include <memory>
3#include <string>
4#include <vector>
5
6#include "envoy/config/core/v3/base.pb.h"
7#include "envoy/config/core/v3/health_check.pb.h"
8#include "envoy/config/core/v3/health_check.pb.validate.h"
9#include "envoy/config/endpoint/v3/endpoint_components.pb.h"
10#include "envoy/data/core/v3/health_check_event.pb.h"
11
12#include "common/buffer/buffer_impl.h"
13#include "common/buffer/zero_copy_input_stream_impl.h"
14#include "common/grpc/common.h"
15#include "common/http/headers.h"
16#include "common/json/json_loader.h"
17#include "common/network/utility.h"
18#include "common/protobuf/utility.h"
19#include "common/upstream/health_checker_impl.h"
20#include "common/upstream/upstream_impl.h"
21
22#include "test/common/http/common.h"
23#include "test/common/upstream/utility.h"
24#include "test/mocks/access_log/mocks.h"
25#include "test/mocks/api/mocks.h"
26#include "test/mocks/network/mocks.h"
27#include "test/mocks/protobuf/mocks.h"
28#include "test/mocks/runtime/mocks.h"
29#include "test/mocks/upstream/mocks.h"
30#include "test/test_common/printers.h"
31#include "test/test_common/simulated_time_system.h"
32#include "test/test_common/utility.h"
33
34#include "gmock/gmock.h"
35#include "gtest/gtest.h"
36
37using testing::_;
38using testing::DoAll;
39using testing::InSequence;
40using testing::Invoke;
41using testing::InvokeWithoutArgs;
42using testing::NiceMock;
43using testing::Return;
44using testing::ReturnRef;
45using testing::SaveArg;
46
47namespace Envoy {
48namespace Upstream {
49namespace {
50
51envoy::config::core::v3::HealthCheck createGrpcHealthCheckConfig() {
52  envoy::config::core::v3::HealthCheck health_check;
53  health_check.mutable_timeout()->set_seconds(1);
54  health_check.mutable_interval()->set_seconds(1);
55  health_check.mutable_unhealthy_threshold()->set_value(2);
56  health_check.mutable_healthy_threshold()->set_value(2);
57  health_check.mutable_grpc_health_check();
58  return health_check;
59}
60
61TEST(HealthCheckerFactoryTest, GrpcHealthCheckHTTP2NotConfiguredException) {
62  NiceMock<Upstream::MockClusterMockPrioritySet> cluster;
63  EXPECT_CALL(*cluster.info_, features()).WillRepeatedly(Return(0));
64
65  Runtime::MockLoader runtime;
66  Runtime::MockRandomGenerator random;
67  Event::MockDispatcher dispatcher;
68  AccessLog::MockAccessLogManager log_manager;
69  NiceMock<ProtobufMessage::MockValidationVisitor> validation_visitor;
70  Api::MockApi api;
71
72  EXPECT_THROW_WITH_MESSAGE(
73      HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime, random,
74                                   dispatcher, log_manager, validation_visitor, api),
75      EnvoyException, "fake_cluster cluster must support HTTP/2 for gRPC healthchecking");
76}
77
78TEST(HealthCheckerFactoryTest, CreateGrpc) {
79
80  NiceMock<Upstream::MockClusterMockPrioritySet> cluster;
81  EXPECT_CALL(*cluster.info_, features())
82      .WillRepeatedly(Return(Upstream::ClusterInfo::Features::HTTP2));
83
84  Runtime::MockLoader runtime;
85  Runtime::MockRandomGenerator random;
86  Event::MockDispatcher dispatcher;
87  AccessLog::MockAccessLogManager log_manager;
88  NiceMock<ProtobufMessage::MockValidationVisitor> validation_visitor;
89  Api::MockApi api;
90
91  EXPECT_NE(nullptr, dynamic_cast<GrpcHealthCheckerImpl*>(
92                         HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster,
93                                                      runtime, random, dispatcher, log_manager,
94                                                      validation_visitor, api)
95                             .get()));
96}
97
98class HealthCheckerTestBase {
99public:
100  std::shared_ptr<MockClusterMockPrioritySet> cluster_{ std::make_shared<NiceMock<MockClusterMockPrioritySet>>()};
101  NiceMock<Event::MockDispatcher> dispatcher_;
102  std::unique_ptr<MockHealthCheckEventLogger> event_logger_{ std::make_unique<NiceMock<MockHealthCheckEventLogger>>()};
103  event_logger_->release();
104  //std::unique_ptr<MockHealthCheckEventLogger> event_logger_{ new MockHealthCheckEventLogger()};
105  NiceMock<Runtime::MockRandomGenerator> random_;
106  NiceMock<Runtime::MockLoader> runtime_;
107};
108
109class TestHttpHealthCheckerImpl : public HttpHealthCheckerImpl {
110public:
111  using HttpHealthCheckerImpl::HttpHealthCheckerImpl;
112
113  Http::CodecClient* createCodecClient(Upstream::Host::CreateConnectionData& conn_data) override {
114    return createCodecClient_(conn_data);
115  };
116
117  // HttpHealthCheckerImpl
118  MOCK_METHOD(Http::CodecClient*, createCodecClient_, (Upstream::Host::CreateConnectionData&));
119
120  Http::CodecClient::Type codecClientType() { return codec_client_type_; }
121};
122
123class HttpHealthCheckerImplTest : public testing::Test, public HealthCheckerTestBase {
124public:
125  struct TestSession {
126    Event::MockTimer* interval_timer_{};
127    Event::MockTimer* timeout_timer_{};
128    Http::MockClientConnection* codec_{};
129    Stats::IsolatedStoreImpl stats_store_;
130    Network::MockClientConnection* client_connection_{};
131    NiceMock<Http::MockRequestEncoder> request_encoder_;
132    Http::ResponseDecoder* stream_response_callbacks_{};
133  };
134
135  using TestSessionPtr = std::unique_ptr<TestSession>;
136  using HostWithHealthCheckMap =
137      std::unordered_map<std::string,
138                         const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig>;
139
140  void allocHealthChecker(const std::string& yaml) {
141    health_checker_ = std::make_shared<TestHttpHealthCheckerImpl>(
142        *cluster_, parseHealthCheckFromV2Yaml(yaml), dispatcher_, runtime_, random_,
143        HealthCheckEventLoggerPtr(event_logger_));
144  }
145
146  void addCompletionCallback() {
147    health_checker_->addHostCheckCompleteCb(
148        [this](HostSharedPtr host, HealthTransition changed_state) -> void {
149          onHostStatus(host, changed_state);
150        });
151  }
152
153  void setupNoServiceValidationHCWithHttp2() {
154    const std::string yaml = R"EOF(
155    timeout: 1s
156    interval: 1s
157    no_traffic_interval: 5s
158    interval_jitter: 1s
159    unhealthy_threshold: 2
160    healthy_threshold: 2
161    http_health_check:
162      service_name_matcher:
163        prefix: locations
164      path: /healthcheck
165      codec_client_type: Http2
166    )EOF";
167
168    allocHealthChecker(yaml);
169    addCompletionCallback();
170  }
171
172  void setupInitialJitter() {
173    const std::string yaml = R"EOF(
174    timeout: 1s
175    interval: 1s
176    no_traffic_interval: 5s
177    initial_jitter: 5s
178    interval_jitter_percent: 40
179    unhealthy_threshold: 2
180    healthy_threshold: 2
181    http_health_check:
182      service_name_matcher:
183        prefix: locations
184      path: /healthcheck
185    )EOF";
186
187    allocHealthChecker(yaml);
188    addCompletionCallback();
189  }
190
191  void setupIntervalJitterPercent() {
192    const std::string yaml = R"EOF(
193    timeout: 1s
194    interval: 1s
195    no_traffic_interval: 5s
196    interval_jitter_percent: 40
197    unhealthy_threshold: 2
198    healthy_threshold: 2
199    http_health_check:
200      service_name_matcher:
201        prefix: locations
202      path: /healthcheck
203    )EOF";
204
205    allocHealthChecker(yaml);
206    addCompletionCallback();
207  }
208
209  void setupNoServiceValidationHC() {
210    const std::string yaml = R"EOF(
211    timeout: 1s
212    interval: 1s
213    no_traffic_interval: 5s
214    interval_jitter: 1s
215    unhealthy_threshold: 2
216    healthy_threshold: 2
217    http_health_check:
218      service_name_matcher:
219        prefix: locations
220      path: /healthcheck
221    )EOF";
222
223    allocHealthChecker(yaml);
224    addCompletionCallback();
225  }
226
227  void setupNoServiceValidationHCOneUnhealthy() {
228    const std::string yaml = R"EOF(
229    timeout: 1s
230    interval: 1s
231    no_traffic_interval: 5s
232    interval_jitter: 1s
233    unhealthy_threshold: 1
234    healthy_threshold: 2
235    http_health_check:
236      service_name_matcher:
237        prefix: locations
238      path: /healthcheck
239    )EOF";
240
241    allocHealthChecker(yaml);
242    addCompletionCallback();
243  }
244
245  void setupNoServiceValidationHCAlwaysLogFailure() {
246    const std::string yaml = R"EOF(
247    timeout: 1s
248    interval: 1s
249    no_traffic_interval: 5s
250    interval_jitter: 1s
251    unhealthy_threshold: 2
252    healthy_threshold: 2
253    http_health_check:
254      service_name_matcher:
255        prefix: locations
256      path: /healthcheck
257    always_log_health_check_failures: true
258    )EOF";
259
260    allocHealthChecker(yaml);
261    addCompletionCallback();
262  }
263
264  void setupNoServiceValidationNoReuseConnectionHC() {
265    std::string yaml = R"EOF(
266    timeout: 1s
267    interval: 1s
268    interval_jitter: 1s
269    unhealthy_threshold: 2
270    healthy_threshold: 2
271    reuse_connection: false
272    http_health_check:
273      path: /healthcheck
274    )EOF";
275
276    allocHealthChecker(yaml);
277    addCompletionCallback();
278  }
279
280  void setupHealthCheckIntervalOverridesHC() {
281    const std::string yaml = R"EOF(
282    timeout: 1s
283    interval: 1s
284    unhealthy_interval: 2s
285    unhealthy_edge_interval: 3s
286    healthy_edge_interval: 4s
287    no_traffic_interval: 5s
288    interval_jitter: 0s
289    unhealthy_threshold: 3
290    healthy_threshold: 3
291    http_health_check:
292      service_name_matcher:
293        prefix: locations
294      path: /healthcheck
295    )EOF";
296
297    allocHealthChecker(yaml);
298    addCompletionCallback();
299  }
300
301  void setupServiceValidationHC() {
302    std::string yaml = R"EOF(
303    timeout: 1s
304    interval: 1s
305    interval_jitter: 1s
306    unhealthy_threshold: 2
307    healthy_threshold: 2
308    http_health_check:
309      service_name_matcher:
310        prefix: locations
311      path: /healthcheck
312    )EOF";
313
314    allocHealthChecker(yaml);
315    addCompletionCallback();
316  }
317
318  void setupDeprecatedServiceNameValidationHC(const std::string& prefix) {
319    std::string yaml = fmt::format(R"EOF(
320    timeout: 1s
321    interval: 1s
322    interval_jitter: 1s
323    unhealthy_threshold: 2
324    healthy_threshold: 2
325    http_health_check:
326      service_name_matcher:
327        prefix: {0}
328      path: /healthcheck
329    )EOF",
330                                   prefix);
331
332    allocHealthChecker(yaml);
333    addCompletionCallback();
334  }
335
336  void setupServicePrefixPatternValidationHC() {
337    std::string yaml = R"EOF(
338    timeout: 1s
339    interval: 1s
340    interval_jitter: 1s
341    unhealthy_threshold: 2
342    healthy_threshold: 2
343    http_health_check:
344      service_name_matcher:
345        prefix: locations
346      path: /healthcheck
347    )EOF";
348
349    allocHealthChecker(yaml);
350    addCompletionCallback();
351  }
352
353  void setupServiceExactPatternValidationHC() {
354    std::string yaml = R"EOF(
355    timeout: 1s
356    interval: 1s
357    interval_jitter: 1s
358    unhealthy_threshold: 2
359    healthy_threshold: 2
360    http_health_check:
361      service_name_matcher:
362        exact: locations-production-iad
363      path: /healthcheck
364    )EOF";
365
366    allocHealthChecker(yaml);
367    addCompletionCallback();
368  }
369
370  void setupServiceRegexPatternValidationHC() {
371    std::string yaml = R"EOF(
372    timeout: 1s
373    interval: 1s
374    interval_jitter: 1s
375    unhealthy_threshold: 2
376    healthy_threshold: 2
377    http_health_check:
378      service_name_matcher:
379        safe_regex:
380          google_re2: {}
381          regex: 'locations-.*-.*$'
382      path: /healthcheck
383    )EOF";
384
385    allocHealthChecker(yaml);
386    addCompletionCallback();
387  }
388
389  void setupServiceValidationWithCustomHostValueHC(const std::string& host) {
390    std::string yaml = fmt::format(R"EOF(
391    timeout: 1s
392    interval: 1s
393    interval_jitter: 1s
394    unhealthy_threshold: 2
395    healthy_threshold: 2
396    http_health_check:
397      service_name_matcher:
398        prefix: locations
399      path: /healthcheck
400      host: {0}
401    )EOF",
402                                   host);
403
404    allocHealthChecker(yaml);
405    addCompletionCallback();
406  }
407
408  const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig
409  makeHealthCheckConfig(const uint32_t port_value) {
410    envoy::config::endpoint::v3::Endpoint::HealthCheckConfig config;
411    config.set_port_value(port_value);
412    return config;
413  }
414
415  void appendTestHosts(std::shared_ptr<MockClusterMockPrioritySet> cluster,
416                       const HostWithHealthCheckMap& hosts, const std::string& protocol = "tcp://",
417                       const uint32_t priority = 0) {
418    for (const auto& host : hosts) {
419      cluster->prioritySet().getMockHostSet(priority)->hosts_.emplace_back(
420          makeTestHost(cluster->info_, fmt::format("{}{}", protocol, host.first), host.second));
421    }
422  }
423
424  void setupServiceValidationWithAdditionalHeaders() {
425    std::string yaml = R"EOF(
426    timeout: 1s
427    interval: 1s
428    interval_jitter: 1s
429    unhealthy_threshold: 2
430    healthy_threshold: 2
431    http_health_check:
432      service_name_matcher:
433        prefix: locations
434      path: /healthcheck
435      host: "www.envoyproxy.io"
436      request_headers_to_add:
437        - header:
438            key: x-envoy-ok
439            value: ok
440        - header:
441            key: x-envoy-cool
442            value: cool
443        - header:
444            key: x-envoy-awesome
445            value: awesome
446        # The following entry replaces the current user-agent.
447        - header:
448            key: user-agent
449            value: CoolEnvoy/HC
450          append: false
451        - header:
452            key: x-protocol
453            value: "%PROTOCOL%"
454        - header:
455            key: x-upstream-metadata
456            value: "%UPSTREAM_METADATA([\"namespace\", \"key\"])%"
457        - header:
458            key: x-downstream-remote-address
459            value: "%DOWNSTREAM_REMOTE_ADDRESS%"
460        - header:
461            key: x-downstream-remote-address-without-port
462            value: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%"
463        - header:
464            key: x-downstream-local-address
465            value: "%DOWNSTREAM_LOCAL_ADDRESS%"
466        - header:
467            key: x-downstream-local-address-without-port
468            value: "%DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT%"
469        - header:
470            key: x-start-time
471            value: "%START_TIME(%s.%9f)%"
472    )EOF";
473
474    allocHealthChecker(yaml);
475    addCompletionCallback();
476  }
477
478  void setupServiceValidationWithoutUserAgent() {
479    std::string yaml = R"EOF(
480    timeout: 1s
481    interval: 1s
482    interval_jitter: 1s
483    unhealthy_threshold: 2
484    healthy_threshold: 2
485    http_health_check:
486      service_name_matcher:
487        prefix: locations
488      path: /healthcheck
489      host: "www.envoyproxy.io"
490      # The following entry removes the default "user-agent" header.
491      request_headers_to_remove: ["user-agent"]
492    )EOF";
493
494    allocHealthChecker(yaml);
495    addCompletionCallback();
496  }
497
498  void expectSessionCreate(const HostWithHealthCheckMap& health_check_map) {
499    // Expectations are in LIFO order.
500    TestSessionPtr new_test_session(new TestSession());
501    test_sessions_.emplace_back(std::move(new_test_session));
502    TestSession& test_session = *test_sessions_.back();
503    test_session.timeout_timer_ = new Event::MockTimer(&dispatcher_);
504    test_session.interval_timer_ = new Event::MockTimer(&dispatcher_);
505    expectClientCreate(test_sessions_.size() - 1, health_check_map);
506  }
507
508  void expectClientCreate(size_t index, const HostWithHealthCheckMap& health_check_map) {
509    TestSession& test_session = *test_sessions_[index];
510    test_session.codec_ = new NiceMock<Http::MockClientConnection>();
511    test_session.client_connection_ = new NiceMock<Network::MockClientConnection>();
512    connection_index_.push_back(index);
513    codec_index_.push_back(index);
514
515    EXPECT_CALL(dispatcher_, createClientConnection_(_, _, _, _))
516        .Times(testing::AnyNumber())
517        .WillRepeatedly(InvokeWithoutArgs([&]() -> Network::ClientConnection* {
518          uint32_t index = connection_index_.front();
519          connection_index_.pop_front();
520          return test_sessions_[index]->client_connection_;
521        }));
522    EXPECT_CALL(*health_checker_, createCodecClient_(_))
523        .WillRepeatedly(
524            Invoke([&](Upstream::Host::CreateConnectionData& conn_data) -> Http::CodecClient* {
525              if (!health_check_map.empty()) {
526                const auto& health_check_config =
527                    health_check_map.at(conn_data.host_description_->address()->asString());
528                // To make sure health checker checks the correct port.
529                EXPECT_EQ(health_check_config.port_value(),
530                          conn_data.host_description_->healthCheckAddress()->ip()->port());
531              }
532              uint32_t index = codec_index_.front();
533              codec_index_.pop_front();
534              TestSession& test_session = *test_sessions_[index];
535              std::shared_ptr<Upstream::MockClusterInfo> cluster{
536                  new NiceMock<Upstream::MockClusterInfo>()};
537              Event::MockDispatcher dispatcher_;
538              return new CodecClientForTest(
539                  Http::CodecClient::Type::HTTP1, std::move(conn_data.connection_),
540                  test_session.codec_, nullptr,
541                  Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000"), dispatcher_);
542            }));
543  }
544
545  void expectStreamCreate(size_t index) {
546    test_sessions_[index]->request_encoder_.stream_.callbacks_.clear();
547    EXPECT_CALL(*test_sessions_[index]->codec_, newStream(_))
548        .WillOnce(DoAll(SaveArgAddress(&test_sessions_[index]->stream_response_callbacks_),
549                        ReturnRef(test_sessions_[index]->request_encoder_)));
550  }
551
552  void respond(size_t index, const std::string& code, bool conn_close, bool proxy_close = false,
553               bool body = false, bool trailers = false,
554               const absl::optional<std::string>& service_cluster = absl::optional<std::string>(),
555               bool degraded = false) {
556    std::unique_ptr<Http::TestResponseHeaderMapImpl> response_headers(
557        new Http::TestResponseHeaderMapImpl{{":status", code}});
558
559    if (degraded) {
560      response_headers->setEnvoyDegraded(1);
561    }
562
563    if (service_cluster) {
564      response_headers->addCopy(Http::Headers::get().EnvoyUpstreamHealthCheckedCluster,
565                                service_cluster.value());
566    }
567    if (conn_close) {
568      response_headers->addCopy("connection", "close");
569    }
570    if (proxy_close) {
571      response_headers->addCopy("proxy-connection", "close");
572    }
573
574    test_sessions_[index]->stream_response_callbacks_->decodeHeaders(std::move(response_headers),
575                                                                     !body && !trailers);
576    if (body) {
577      Buffer::OwnedImpl response_data;
578      test_sessions_[index]->stream_response_callbacks_->decodeData(response_data, !trailers);
579    }
580
581    if (trailers) {
582      test_sessions_[index]->stream_response_callbacks_->decodeTrailers(
583          Http::ResponseTrailerMapPtr{new Http::TestResponseTrailerMapImpl{{"some", "trailer"}}});
584    }
585  }
586
587  void expectSessionCreate() { expectSessionCreate(health_checker_map_); }
588  void expectClientCreate(size_t index) { expectClientCreate(index, health_checker_map_); }
589
590  void expectSuccessStartFailedFailFirst(
591      const absl::optional<std::string>& health_checked_cluster = absl::optional<std::string>()) {
592    cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
593        makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
594    cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet(
595        Host::HealthFlag::FAILED_ACTIVE_HC);
596    expectSessionCreate();
597    expectStreamCreate(0);
598    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
599    health_checker_->start();
600
601    // Test that failing first disables fast success.
602    EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
603    EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
604    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
605    EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
606    respond(0, "503", false, false, false, false, health_checked_cluster);
607    EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
608        Host::HealthFlag::FAILED_ACTIVE_HC));
609    EXPECT_EQ(Host::Health::Unhealthy,
610              cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
611
612    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
613    expectStreamCreate(0);
614    test_sessions_[0]->interval_timer_->invokeCallback();
615
616    EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
617    EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
618    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
619    respond(0, "200", false, false, false, false, health_checked_cluster);
620    EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
621        Host::HealthFlag::FAILED_ACTIVE_HC));
622    EXPECT_EQ(Host::Health::Unhealthy,
623              cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
624
625    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
626    expectStreamCreate(0);
627    test_sessions_[0]->interval_timer_->invokeCallback();
628
629    EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
630    EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
631    EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
632    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
633    respond(0, "200", false, false, false, false, health_checked_cluster);
634    EXPECT_EQ(Host::Health::Healthy,
635              cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
636  }
637
638  MOCK_METHOD(void, onHostStatus, (HostSharedPtr host, HealthTransition changed_state));
639
640  std::vector<TestSessionPtr> test_sessions_;
641  std::shared_ptr<TestHttpHealthCheckerImpl> health_checker_;
642  std::list<uint32_t> connection_index_{};
643  std::list<uint32_t> codec_index_{};
644  const HostWithHealthCheckMap health_checker_map_{};
645};
646
647TEST_F(HttpHealthCheckerImplTest, Success) {
648  setupNoServiceValidationHC();
649  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
650
651  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
652      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
653  cluster_->info_->stats().upstream_cx_total_.inc();
654  expectSessionCreate();
655  expectStreamCreate(0);
656  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
657  health_checker_->start();
658
659  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
660  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
661      .WillOnce(Return(45000));
662  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
663              enableTimer(std::chrono::milliseconds(45000), _));
664  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
665  respond(0, "200", false, false, true);
666  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
667}
668
669TEST_F(HttpHealthCheckerImplTest, Degraded) {
670  setupNoServiceValidationHC();
671  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)).Times(2);
672
673  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
674      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
675  cluster_->info_->stats().upstream_cx_total_.inc();
676  expectSessionCreate();
677  expectStreamCreate(0);
678  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
679  health_checker_->start();
680
681  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
682  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
683      .WillRepeatedly(Return(45000));
684
685  // We start off as healthy, and should go degraded after receiving the degraded health response.
686  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
687  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
688  EXPECT_CALL(*event_logger_, logDegraded(_, _));
689  respond(0, "200", false, false, true, false, {}, true);
690  EXPECT_EQ(Host::Health::Degraded, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
691
692  // Then, after receiving a regular health check response we should go back to healthy.
693  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
694  expectStreamCreate(0);
695  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
696  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
697  test_sessions_[0]->interval_timer_->invokeCallback();
698  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
699  EXPECT_CALL(*event_logger_, logNoLongerDegraded(_, _));
700  respond(0, "200", false, false, true, false, {}, false);
701  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
702}
703
704TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitter) {
705  setupNoServiceValidationHC();
706  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(testing::AnyNumber());
707
708  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
709      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
710  expectSessionCreate();
711  expectStreamCreate(0);
712  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
713  health_checker_->start();
714
715  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
716  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
717  respond(0, "200", false, false, true, true);
718  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
719
720  for (int i = 0; i < 50000; i += 239) {
721    EXPECT_CALL(random_, random()).WillOnce(Return(i));
722    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
723    expectStreamCreate(0);
724    test_sessions_[0]->interval_timer_->invokeCallback();
725    // the jitter is 1000ms here
726    EXPECT_CALL(*test_sessions_[0]->interval_timer_,
727                enableTimer(std::chrono::milliseconds(5000 + i % 1000), _));
728    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
729    respond(0, "200", false, false, true, true);
730  }
731}
732
733TEST_F(HttpHealthCheckerImplTest, InitialJitterNoTraffic) {
734  setupInitialJitter();
735  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(testing::AnyNumber());
736
737  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
738      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
739  expectSessionCreate();
740  expectStreamCreate(0);
741  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
742  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
743  health_checker_->start();
744  test_sessions_[0]->interval_timer_->invokeCallback();
745
746  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
747  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
748  respond(0, "200", false, false, true, true);
749  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
750
751  for (int i = 0; i < 2; i += 1) {
752    EXPECT_CALL(random_, random()).WillOnce(Return(i));
753    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
754    expectStreamCreate(0);
755    test_sessions_[0]->interval_timer_->invokeCallback();
756    // the jitter is 40% of 5000, so should be 2000
757    EXPECT_CALL(*test_sessions_[0]->interval_timer_,
758                enableTimer(std::chrono::milliseconds(5000 + i % 2000), _));
759    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
760    respond(0, "200", false, false, true, true);
761  }
762}
763
764TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitterPercentNoTraffic) {
765  setupIntervalJitterPercent();
766  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(testing::AnyNumber());
767
768  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
769      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
770  expectSessionCreate();
771  expectStreamCreate(0);
772  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
773  health_checker_->start();
774
775  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
776  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
777  respond(0, "200", false, false, true, true);
778  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
779
780  for (int i = 0; i < 50000; i += 239) {
781    EXPECT_CALL(random_, random()).WillOnce(Return(i));
782    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
783    expectStreamCreate(0);
784    test_sessions_[0]->interval_timer_->invokeCallback();
785    // the jitter is 40% of 5000, so should be 2000
786    EXPECT_CALL(*test_sessions_[0]->interval_timer_,
787                enableTimer(std::chrono::milliseconds(5000 + i % 2000), _));
788    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
789    respond(0, "200", false, false, true, true);
790  }
791}
792
793TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitterPercent) {
794  setupIntervalJitterPercent();
795  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(testing::AnyNumber());
796
797  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
798      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
799  cluster_->info_->stats().upstream_cx_total_.inc();
800  expectSessionCreate();
801  expectStreamCreate(0);
802  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
803  health_checker_->start();
804
805  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
806  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
807  respond(0, "200", false, false, true, true);
808  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
809
810  for (int i = 0; i < 50000; i += 239) {
811    EXPECT_CALL(random_, random()).WillOnce(Return(i));
812    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
813    expectStreamCreate(0);
814    test_sessions_[0]->interval_timer_->invokeCallback();
815    // the jitter is 40% of 1000, so should be 400
816    EXPECT_CALL(*test_sessions_[0]->interval_timer_,
817                enableTimer(std::chrono::milliseconds(1000 + i % 400), _));
818    EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
819    respond(0, "200", false, false, true, true);
820  }
821}
822
823TEST_F(HttpHealthCheckerImplTest, SuccessWithSpurious100Continue) {
824  setupNoServiceValidationHC();
825  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
826
827  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
828      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
829  cluster_->info_->stats().upstream_cx_total_.inc();
830  expectSessionCreate();
831  expectStreamCreate(0);
832  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
833  health_checker_->start();
834
835  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
836  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
837      .WillOnce(Return(45000));
838  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
839              enableTimer(std::chrono::milliseconds(45000), _));
840  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
841
842  std::unique_ptr<Http::TestResponseHeaderMapImpl> continue_headers(
843      new Http::TestResponseHeaderMapImpl{{":status", "100"}});
844  test_sessions_[0]->stream_response_callbacks_->decode100ContinueHeaders(
845      std::move(continue_headers));
846
847  respond(0, "200", false, false, true);
848  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
849}
850
851TEST_F(HttpHealthCheckerImplTest, SuccessWithSpuriousMetadata) {
852  setupNoServiceValidationHC();
853  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
854
855  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
856      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
857  cluster_->info_->stats().upstream_cx_total_.inc();
858  expectSessionCreate();
859  expectStreamCreate(0);
860  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
861  health_checker_->start();
862
863  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
864  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
865      .WillOnce(Return(45000));
866  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
867              enableTimer(std::chrono::milliseconds(45000), _));
868  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
869
870  std::unique_ptr<Http::MetadataMap> metadata_map(new Http::MetadataMap());
871  metadata_map->insert(std::make_pair<std::string, std::string>("key", "value"));
872  test_sessions_[0]->stream_response_callbacks_->decodeMetadata(std::move(metadata_map));
873
874  respond(0, "200", false, false, true);
875  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
876}
877
878// Test host check success with multiple hosts.
879TEST_F(HttpHealthCheckerImplTest, SuccessWithMultipleHosts) {
880  setupNoServiceValidationHC();
881  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(2);
882
883  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
884      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80"),
885      makeTestHost(cluster_->info_, "tcp://127.0.0.1:81")};
886  cluster_->info_->stats().upstream_cx_total_.inc();
887  cluster_->info_->stats().upstream_cx_total_.inc();
888  expectSessionCreate();
889  expectStreamCreate(0);
890  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
891  expectSessionCreate();
892  expectStreamCreate(1);
893  EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_, _));
894  health_checker_->start();
895
896  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).Times(2);
897  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
898      .Times(2)
899      .WillRepeatedly(Return(45000));
900  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
901              enableTimer(std::chrono::milliseconds(45000), _));
902  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
903  EXPECT_CALL(*test_sessions_[1]->interval_timer_,
904              enableTimer(std::chrono::milliseconds(45000), _));
905  EXPECT_CALL(*test_sessions_[1]->timeout_timer_, disableTimer());
906  respond(0, "200", false, false, true);
907  respond(1, "200", false, false, true);
908  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
909  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[1]->health());
910}
911
912// Test host check success with multiple hosts across multiple priorities.
913TEST_F(HttpHealthCheckerImplTest, SuccessWithMultipleHostSets) {
914  setupNoServiceValidationHC();
915  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(2);
916
917  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
918      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
919  cluster_->prioritySet().getMockHostSet(1)->hosts_ = {
920      makeTestHost(cluster_->info_, "tcp://127.0.0.1:81")};
921  cluster_->info_->stats().upstream_cx_total_.inc();
922  cluster_->info_->stats().upstream_cx_total_.inc();
923  expectSessionCreate();
924  expectStreamCreate(0);
925  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
926  expectSessionCreate();
927  expectStreamCreate(1);
928  EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_, _));
929  health_checker_->start();
930
931  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).Times(2);
932  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
933      .Times(2)
934      .WillRepeatedly(Return(45000));
935  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
936              enableTimer(std::chrono::milliseconds(45000), _));
937  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
938  EXPECT_CALL(*test_sessions_[1]->interval_timer_,
939              enableTimer(std::chrono::milliseconds(45000), _));
940  EXPECT_CALL(*test_sessions_[1]->timeout_timer_, disableTimer());
941  respond(0, "200", false, false, true);
942  respond(1, "200", false, false, true);
943  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
944  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(1)->hosts_[0]->health());
945}
946
947// Validate that runtime settings can't force a zero lengthy retry duration (and hence livelock).
948TEST_F(HttpHealthCheckerImplTest, ZeroRetryInterval) {
949  const std::string host = "fake_cluster";
950  const std::string path = "/healthcheck";
951  const std::string yaml = R"EOF(
952    timeout: 1s
953    interval: 1s
954    no_traffic_interval: 1s
955    interval_jitter_percent: 40
956    unhealthy_threshold: 2
957    healthy_threshold: 2
958    http_health_check:
959      service_name_matcher:
960        prefix: locations
961      path: /healthcheck
962    )EOF";
963
964  allocHealthChecker(yaml);
965  addCompletionCallback();
966
967  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
968      .WillOnce(Return(true));
969
970  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
971
972  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
973      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
974  cluster_->info_->stats().upstream_cx_total_.inc();
975  expectSessionCreate();
976  expectStreamCreate(0);
977  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
978  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
979      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
980        EXPECT_EQ(headers.Host()->value().getStringView(), host);
981        EXPECT_EQ(headers.Path()->value().getStringView(), path);
982        EXPECT_EQ(headers.Scheme()->value().getStringView(),
983                  Http::Headers::get().SchemeValues.Http);
984      }));
985  health_checker_->start();
986
987  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).WillOnce(Return(0));
988  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)).WillOnce(Return(0));
989  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1), _));
990  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
991  absl::optional<std::string> health_checked_cluster("locations-production-iad");
992  respond(0, "200", false, false, true, false, health_checked_cluster);
993  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
994}
995
996MATCHER_P(ApplicationProtocolListEq, expected, "") {
997  const Network::TransportSocketOptionsSharedPtr& options = arg;
998  EXPECT_EQ(options->applicationProtocolListOverride(), std::vector<std::string>{expected});
999  return true;
1000}
1001
1002TEST_F(HttpHealthCheckerImplTest, TlsOptions) {
1003  const std::string host = "fake_cluster";
1004  const std::string path = "/healthcheck";
1005  const std::string yaml = R"EOF(
1006    timeout: 1s
1007    interval: 1s
1008    no_traffic_interval: 1s
1009    interval_jitter_percent: 40
1010    unhealthy_threshold: 2
1011    healthy_threshold: 2
1012    http_health_check:
1013      service_name_matcher:
1014        prefix: locations
1015      path: /healthcheck
1016    tls_options:
1017      alpn_protocols:
1018      - http1
1019    )EOF";
1020
1021  auto socket_factory = new Network::MockTransportSocketFactory();
1022  EXPECT_CALL(*socket_factory, implementsSecureTransport()).WillOnce(Return(true));
1023  auto transport_socket_match = new NiceMock<Upstream::MockTransportSocketMatcher>(
1024      Network::TransportSocketFactoryPtr(socket_factory));
1025  cluster_->info_->transport_socket_matcher_.reset(transport_socket_match);
1026
1027  EXPECT_CALL(*socket_factory, createTransportSocket(ApplicationProtocolListEq("http1")));
1028
1029  allocHealthChecker(yaml);
1030  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1031      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1032  cluster_->info_->stats().upstream_cx_total_.inc();
1033  expectSessionCreate();
1034  expectStreamCreate(0);
1035  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1036  health_checker_->start();
1037}
1038
1039TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheck) {
1040  const std::string host = "fake_cluster";
1041  const std::string path = "/healthcheck";
1042  setupServiceValidationHC();
1043  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1044      .WillOnce(Return(true));
1045
1046  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1047
1048  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1049      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1050  cluster_->info_->stats().upstream_cx_total_.inc();
1051  expectSessionCreate();
1052  expectStreamCreate(0);
1053  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1054  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1055      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1056        EXPECT_EQ(headers.Host()->value().getStringView(), host);
1057        EXPECT_EQ(headers.Path()->value().getStringView(), path);
1058        EXPECT_EQ(headers.Scheme()->value().getStringView(),
1059                  Http::Headers::get().SchemeValues.Http);
1060      }));
1061  health_checker_->start();
1062
1063  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1064  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1065      .WillOnce(Return(45000));
1066  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1067              enableTimer(std::chrono::milliseconds(45000), _));
1068  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1069  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1070  respond(0, "200", false, false, true, false, health_checked_cluster);
1071  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1072}
1073
1074TEST_F(HttpHealthCheckerImplTest, SuccessServicePrefixPatternCheck) {
1075  const std::string host = "fake_cluster";
1076  const std::string path = "/healthcheck";
1077  setupServicePrefixPatternValidationHC();
1078  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1079      .WillOnce(Return(true));
1080
1081  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1082
1083  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1084      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1085  cluster_->info_->stats().upstream_cx_total_.inc();
1086  expectSessionCreate();
1087  expectStreamCreate(0);
1088  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1089  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1090      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1091        EXPECT_EQ(headers.Host()->value().getStringView(), host);
1092        EXPECT_EQ(headers.Path()->value().getStringView(), path);
1093        EXPECT_EQ(headers.Scheme()->value().getStringView(),
1094                  Http::Headers::get().SchemeValues.Http);
1095      }));
1096  health_checker_->start();
1097
1098  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1099  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1100      .WillOnce(Return(45000));
1101  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1102              enableTimer(std::chrono::milliseconds(45000), _));
1103  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1104  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1105  respond(0, "200", false, false, true, false, health_checked_cluster);
1106  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1107}
1108
1109TEST_F(HttpHealthCheckerImplTest, SuccessServiceExactPatternCheck) {
1110  const std::string host = "fake_cluster";
1111  const std::string path = "/healthcheck";
1112  setupServiceExactPatternValidationHC();
1113  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1114      .WillOnce(Return(true));
1115
1116  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1117
1118  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1119      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1120  cluster_->info_->stats().upstream_cx_total_.inc();
1121  expectSessionCreate();
1122  expectStreamCreate(0);
1123  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1124  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1125      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1126        EXPECT_EQ(headers.Host()->value().getStringView(), host);
1127        EXPECT_EQ(headers.Path()->value().getStringView(), path);
1128        EXPECT_EQ(headers.Scheme()->value().getStringView(),
1129                  Http::Headers::get().SchemeValues.Http);
1130      }));
1131  health_checker_->start();
1132
1133  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1134  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1135      .WillOnce(Return(45000));
1136  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1137              enableTimer(std::chrono::milliseconds(45000), _));
1138  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1139  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1140  respond(0, "200", false, false, true, false, health_checked_cluster);
1141  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1142}
1143
1144TEST_F(HttpHealthCheckerImplTest, SuccessServiceRegexPatternCheck) {
1145  const std::string host = "fake_cluster";
1146  const std::string path = "/healthcheck";
1147  setupServiceRegexPatternValidationHC();
1148  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1149      .WillOnce(Return(true));
1150
1151  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1152
1153  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1154      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1155  cluster_->info_->stats().upstream_cx_total_.inc();
1156  expectSessionCreate();
1157  expectStreamCreate(0);
1158  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1159  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1160      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1161        EXPECT_EQ(headers.Host()->value().getStringView(), host);
1162        EXPECT_EQ(headers.Path()->value().getStringView(), path);
1163        EXPECT_EQ(headers.Scheme()->value().getStringView(),
1164                  Http::Headers::get().SchemeValues.Http);
1165      }));
1166  health_checker_->start();
1167
1168  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1169  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1170      .WillOnce(Return(45000));
1171  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1172              enableTimer(std::chrono::milliseconds(45000), _));
1173  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1174  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1175  respond(0, "200", false, false, true, false, health_checked_cluster);
1176  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1177}
1178
1179// This test verifies that when a hostname is set in the endpoint's HealthCheckConfig, it is used in
1180// the health check request.
1181TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithCustomHostValueOnTheHost) {
1182  const std::string host = "www.envoyproxy.io";
1183  envoy::config::endpoint::v3::Endpoint::HealthCheckConfig health_check_config;
1184  health_check_config.set_hostname(host);
1185  auto test_host = std::make_shared<HostImpl>(
1186      cluster_->info_, "", Network::Utility::resolveUrl("tcp://127.0.0.1:80"), nullptr, 1,
1187      envoy::config::core::v3::Locality(), health_check_config, 0,
1188      envoy::config::core::v3::UNKNOWN);
1189  const std::string path = "/healthcheck";
1190  setupServiceValidationHC();
1191  // Requires non-empty `service_name` in config.
1192  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1193      .WillOnce(Return(true));
1194
1195  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1196
1197  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {test_host};
1198  cluster_->info_->stats().upstream_cx_total_.inc();
1199  expectSessionCreate();
1200  expectStreamCreate(0);
1201  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1202  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1203      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1204        EXPECT_EQ(headers.Host()->value().getStringView(), host);
1205        EXPECT_EQ(headers.Path()->value().getStringView(), path);
1206      }));
1207  health_checker_->start();
1208
1209  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1210  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1211      .WillOnce(Return(45000));
1212  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1213              enableTimer(std::chrono::milliseconds(45000), _));
1214  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1215  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1216  respond(0, "200", false, false, true, false, health_checked_cluster);
1217  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1218}
1219
1220// This test verifies that when a hostname is set in the endpoint's HealthCheckConfig and in the
1221// cluster level configuration, the one in the endpoint takes priority.
1222TEST_F(HttpHealthCheckerImplTest,
1223       SuccessServiceCheckWithCustomHostValueOnTheHostThatOverridesConfigValue) {
1224  const std::string host = "www.envoyproxy.io";
1225  envoy::config::endpoint::v3::Endpoint::HealthCheckConfig health_check_config;
1226  health_check_config.set_hostname(host);
1227  auto test_host = std::make_shared<HostImpl>(
1228      cluster_->info_, "", Network::Utility::resolveUrl("tcp://127.0.0.1:80"), nullptr, 1,
1229      envoy::config::core::v3::Locality(), health_check_config, 0,
1230      envoy::config::core::v3::UNKNOWN);
1231  const std::string path = "/healthcheck";
1232  // Setup health check config with a different host, to check that we still get the host configured
1233  // on the endpoint.
1234  setupServiceValidationWithCustomHostValueHC("foo.com");
1235  // Requires non-empty `service_name` in config.
1236  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1237      .WillOnce(Return(true));
1238
1239  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1240
1241  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {test_host};
1242  cluster_->info_->stats().upstream_cx_total_.inc();
1243  expectSessionCreate();
1244  expectStreamCreate(0);
1245  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1246  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1247      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1248        EXPECT_EQ(headers.Host()->value().getStringView(), host);
1249        EXPECT_EQ(headers.Path()->value().getStringView(), path);
1250      }));
1251  health_checker_->start();
1252
1253  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1254  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1255      .WillOnce(Return(45000));
1256  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1257              enableTimer(std::chrono::milliseconds(45000), _));
1258  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1259  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1260  respond(0, "200", false, false, true, false, health_checked_cluster);
1261  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1262}
1263
1264TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithCustomHostValue) {
1265  const std::string host = "www.envoyproxy.io";
1266  const std::string path = "/healthcheck";
1267  setupServiceValidationWithCustomHostValueHC(host);
1268  // Requires non-empty `service_name` in config.
1269  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1270      .WillOnce(Return(true));
1271
1272  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1273
1274  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1275      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1276  cluster_->info_->stats().upstream_cx_total_.inc();
1277  expectSessionCreate();
1278  expectStreamCreate(0);
1279  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1280  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1281      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1282        EXPECT_EQ(headers.Host()->value().getStringView(), host);
1283        EXPECT_EQ(headers.Path()->value().getStringView(), path);
1284      }));
1285  health_checker_->start();
1286
1287  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1288  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1289      .WillOnce(Return(45000));
1290  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1291              enableTimer(std::chrono::milliseconds(45000), _));
1292  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1293  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1294  respond(0, "200", false, false, true, false, health_checked_cluster);
1295  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1296}
1297
1298TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithAdditionalHeaders) {
1299  const Http::LowerCaseString header_ok("x-envoy-ok");
1300  const Http::LowerCaseString header_cool("x-envoy-cool");
1301  const Http::LowerCaseString header_awesome("x-envoy-awesome");
1302  const Http::LowerCaseString upstream_metadata("x-upstream-metadata");
1303  const Http::LowerCaseString protocol("x-protocol");
1304  const Http::LowerCaseString downstream_remote_address("x-downstream-remote-address");
1305  const Http::LowerCaseString downstream_remote_address_without_port(
1306      "x-downstream-remote-address-without-port");
1307  const Http::LowerCaseString downstream_local_address("x-downstream-local-address");
1308  const Http::LowerCaseString downstream_local_address_without_port(
1309      "x-downstream-local-address-without-port");
1310  const Http::LowerCaseString start_time("x-start-time");
1311
1312  const std::string value_ok = "ok";
1313  const std::string value_cool = "cool";
1314  const std::string value_awesome = "awesome";
1315
1316  const std::string value_user_agent = "CoolEnvoy/HC";
1317  const std::string value_upstream_metadata = "value";
1318  const std::string value_protocol = "HTTP/1.1";
1319  const std::string value_downstream_remote_address = "127.0.0.1:0";
1320  const std::string value_downstream_remote_address_without_port = "127.0.0.1";
1321  const std::string value_downstream_local_address = "127.0.0.1:0";
1322  const std::string value_downstream_local_address_without_port = "127.0.0.1";
1323
1324  setupServiceValidationWithAdditionalHeaders();
1325  // Requires non-empty `service_name` in config.
1326  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1327      .WillOnce(Return(true));
1328
1329  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1330  auto metadata = TestUtility::parseYaml<envoy::config::core::v3::Metadata>(
1331      R"EOF(
1332        filter_metadata:
1333          namespace:
1334            key: value
1335      )EOF");
1336
1337  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1338      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80", metadata)};
1339  cluster_->info_->stats().upstream_cx_total_.inc();
1340  expectSessionCreate();
1341  expectStreamCreate(0);
1342  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1343  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1344      .WillRepeatedly(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1345        EXPECT_EQ(headers.get(header_ok)->value().getStringView(), value_ok);
1346        EXPECT_EQ(headers.get(header_cool)->value().getStringView(), value_cool);
1347        EXPECT_EQ(headers.get(header_awesome)->value().getStringView(), value_awesome);
1348
1349        EXPECT_EQ(headers.UserAgent()->value().getStringView(), value_user_agent);
1350        EXPECT_EQ(headers.get(upstream_metadata)->value().getStringView(), value_upstream_metadata);
1351
1352        EXPECT_EQ(headers.get(protocol)->value().getStringView(), value_protocol);
1353        EXPECT_EQ(headers.get(downstream_remote_address)->value().getStringView(),
1354                  value_downstream_remote_address);
1355        EXPECT_EQ(headers.get(downstream_remote_address_without_port)->value().getStringView(),
1356                  value_downstream_remote_address_without_port);
1357        EXPECT_EQ(headers.get(downstream_local_address)->value().getStringView(),
1358                  value_downstream_local_address);
1359        EXPECT_EQ(headers.get(downstream_local_address_without_port)->value().getStringView(),
1360                  value_downstream_local_address_without_port);
1361
1362        Envoy::DateFormatter date_formatter("%s.%9f");
1363        std::string current_start_time =
1364            date_formatter.fromTime(dispatcher_.timeSource().systemTime());
1365        EXPECT_EQ(headers.get(start_time)->value().getStringView(), current_start_time);
1366      }));
1367  health_checker_->start();
1368
1369  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1370  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1371      .WillOnce(Return(45000));
1372  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1373              enableTimer(std::chrono::milliseconds(45000), _));
1374  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1375  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1376  respond(0, "200", false, false, true, false, health_checked_cluster);
1377  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1378
1379  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1380  expectStreamCreate(0);
1381  test_sessions_[0]->interval_timer_->invokeCallback();
1382}
1383
1384TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithoutUserAgent) {
1385  setupServiceValidationWithoutUserAgent();
1386  // Requires non-empty `service_name` in config.
1387  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1388      .WillOnce(Return(true));
1389
1390  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1391  auto metadata = TestUtility::parseYaml<envoy::config::core::v3::Metadata>(
1392      R"EOF(
1393        filter_metadata:
1394          namespace:
1395            key: value
1396      )EOF");
1397
1398  std::string current_start_time;
1399  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1400      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80", metadata)};
1401  cluster_->info_->stats().upstream_cx_total_.inc();
1402  expectSessionCreate();
1403  expectStreamCreate(0);
1404  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1405  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
1406      .WillRepeatedly(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
1407        EXPECT_EQ(headers.UserAgent(), nullptr);
1408      }));
1409  health_checker_->start();
1410
1411  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1412  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1413      .WillOnce(Return(45000));
1414  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1415              enableTimer(std::chrono::milliseconds(45000), _));
1416  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1417  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1418  respond(0, "200", false, false, true, false, health_checked_cluster);
1419  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1420
1421  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1422  expectStreamCreate(0);
1423  test_sessions_[0]->interval_timer_->invokeCallback();
1424}
1425
1426TEST_F(HttpHealthCheckerImplTest, ServiceDoesNotMatchFail) {
1427  setupServiceValidationHC();
1428  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1429  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1430      .WillOnce(Return(true));
1431
1432  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)).Times(1);
1433  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1434
1435  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1436      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1437  cluster_->info_->stats().upstream_cx_total_.inc();
1438  expectSessionCreate();
1439  expectStreamCreate(0);
1440  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1441  health_checker_->start();
1442
1443  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1444  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1445      .WillOnce(Return(45000));
1446  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1447              enableTimer(std::chrono::milliseconds(45000), _));
1448  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1449  absl::optional<std::string> health_checked_cluster("api-production-iad");
1450  respond(0, "200", false, false, true, false, health_checked_cluster);
1451  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1452      Host::HealthFlag::FAILED_ACTIVE_HC));
1453  EXPECT_EQ(Host::Health::Unhealthy,
1454            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1455}
1456
1457TEST_F(HttpHealthCheckerImplTest, ServicePatternDoesNotMatchFail) {
1458  setupServiceRegexPatternValidationHC();
1459  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1460  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1461      .WillOnce(Return(true));
1462
1463  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)).Times(1);
1464  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1465
1466  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1467      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1468  cluster_->info_->stats().upstream_cx_total_.inc();
1469  expectSessionCreate();
1470  expectStreamCreate(0);
1471  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1472  health_checker_->start();
1473
1474  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1475  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1476      .WillOnce(Return(45000));
1477  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1478              enableTimer(std::chrono::milliseconds(45000), _));
1479  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1480  absl::optional<std::string> health_checked_cluster("api-production-iad");
1481  respond(0, "200", false, false, true, false, health_checked_cluster);
1482  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1483      Host::HealthFlag::FAILED_ACTIVE_HC));
1484  EXPECT_EQ(Host::Health::Unhealthy,
1485            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1486}
1487
1488TEST_F(HttpHealthCheckerImplTest, ServiceNotPresentInResponseFail) {
1489  setupServiceValidationHC();
1490  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1491  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1492      .WillOnce(Return(true));
1493
1494  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)).Times(1);
1495  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1496
1497  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1498      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1499  cluster_->info_->stats().upstream_cx_total_.inc();
1500  expectSessionCreate();
1501  expectStreamCreate(0);
1502  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1503  health_checker_->start();
1504
1505  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1506  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1507      .WillOnce(Return(45000));
1508  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1509              enableTimer(std::chrono::milliseconds(45000), _));
1510  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1511  respond(0, "200", false, false, true, false);
1512  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1513      Host::HealthFlag::FAILED_ACTIVE_HC));
1514  EXPECT_EQ(Host::Health::Unhealthy,
1515            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1516}
1517
1518TEST_F(HttpHealthCheckerImplTest, ServiceCheckRuntimeOff) {
1519  setupServiceValidationHC();
1520  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1521      .WillOnce(Return(false));
1522
1523  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1524
1525  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1526      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1527  cluster_->info_->stats().upstream_cx_total_.inc();
1528  expectSessionCreate();
1529  expectStreamCreate(0);
1530  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1531  health_checker_->start();
1532
1533  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1534  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1535      .WillOnce(Return(45000));
1536  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1537              enableTimer(std::chrono::milliseconds(45000), _));
1538  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1539  absl::optional<std::string> health_checked_cluster("api-production-iad");
1540  respond(0, "200", false, false, true, false, health_checked_cluster);
1541  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1542}
1543
1544TEST_F(HttpHealthCheckerImplTest, ServiceCheckRuntimeOffWithStringPattern) {
1545  setupServicePrefixPatternValidationHC();
1546  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1547      .WillOnce(Return(false));
1548
1549  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1550
1551  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1552      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1553  cluster_->info_->stats().upstream_cx_total_.inc();
1554  expectSessionCreate();
1555  expectStreamCreate(0);
1556  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1557  health_checker_->start();
1558
1559  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
1560  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
1561      .WillOnce(Return(45000));
1562  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
1563              enableTimer(std::chrono::milliseconds(45000), _));
1564  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1565  absl::optional<std::string> health_checked_cluster("api-production-iad");
1566  respond(0, "200", false, false, true, false, health_checked_cluster);
1567  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1568}
1569
1570TEST_F(HttpHealthCheckerImplTest, SuccessStartFailedFailFirstServiceCheck) {
1571  setupNoServiceValidationHC();
1572  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
1573      .WillRepeatedly(Return(true));
1574  absl::optional<std::string> health_checked_cluster("locations-production-iad");
1575  expectSuccessStartFailedFailFirst(health_checked_cluster);
1576}
1577
1578TEST_F(HttpHealthCheckerImplTest, SuccessNoTraffic) {
1579  setupNoServiceValidationHC();
1580  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
1581
1582  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1583      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1584  expectSessionCreate();
1585  expectStreamCreate(0);
1586  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1587  health_checker_->start();
1588
1589  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000), _));
1590  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1591  respond(0, "200", false, false, true, true);
1592  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1593}
1594
1595TEST_F(HttpHealthCheckerImplTest, SuccessStartFailedSuccessFirst) {
1596  setupNoServiceValidationHC();
1597  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1598      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1599  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet(
1600      Host::HealthFlag::FAILED_ACTIVE_HC);
1601  expectSessionCreate();
1602  expectStreamCreate(0);
1603  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1604  health_checker_->start();
1605
1606  // Test fast success immediately moves us to healthy.
1607  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)).Times(1);
1608  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, true));
1609  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).WillOnce(Return(500));
1610  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _));
1611  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(500), _));
1612  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1613  respond(0, "200", false);
1614  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1615}
1616
1617TEST_F(HttpHealthCheckerImplTest, SuccessStartFailedFailFirst) {
1618  setupNoServiceValidationHC();
1619  expectSuccessStartFailedFailFirst();
1620}
1621
1622TEST_F(HttpHealthCheckerImplTest, SuccessStartFailedFailFirstLogError) {
1623  setupNoServiceValidationHCAlwaysLogFailure();
1624  expectSuccessStartFailedFailFirst();
1625}
1626
1627// Verify that removal during a failure callback works.
1628TEST_F(HttpHealthCheckerImplTest, HttpFailRemoveHostInCallbackNoClose) {
1629  setupNoServiceValidationHC();
1630  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1631      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1632  expectSessionCreate();
1633  expectStreamCreate(0);
1634  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1635  health_checker_->start();
1636
1637  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed))
1638      .WillOnce(Invoke([&](HostSharedPtr host, HealthTransition) {
1639        cluster_->prioritySet().getMockHostSet(0)->hosts_ = {};
1640        cluster_->prioritySet().runUpdateCallbacks(0, {}, {host});
1641      }));
1642  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1643  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)).Times(0);
1644  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()).Times(0);
1645  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1646  respond(0, "503", false);
1647}
1648
1649// Verify that removal during a failure callback works with connection close.
1650TEST_F(HttpHealthCheckerImplTest, HttpFailRemoveHostInCallbackClose) {
1651  setupNoServiceValidationHC();
1652  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1653      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1654  expectSessionCreate();
1655  expectStreamCreate(0);
1656  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1657  health_checker_->start();
1658
1659  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed))
1660      .WillOnce(Invoke([&](HostSharedPtr host, HealthTransition) {
1661        cluster_->prioritySet().getMockHostSet(0)->hosts_ = {};
1662        cluster_->prioritySet().runUpdateCallbacks(0, {}, {host});
1663      }));
1664  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1665  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)).Times(0);
1666  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()).Times(0);
1667  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1668  respond(0, "503", true);
1669}
1670
1671TEST_F(HttpHealthCheckerImplTest, HttpFail) {
1672  setupNoServiceValidationHC();
1673  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1674      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1675  expectSessionCreate();
1676  expectStreamCreate(0);
1677  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1678  health_checker_->start();
1679
1680  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
1681  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1682  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1683  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1684  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1685  respond(0, "503", false);
1686  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1687      Host::HealthFlag::FAILED_ACTIVE_HC));
1688  EXPECT_EQ(Host::Health::Unhealthy,
1689            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1690
1691  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
1692            Host::ActiveHealthFailureType::UNHEALTHY);
1693  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1694  expectStreamCreate(0);
1695  test_sessions_[0]->interval_timer_->invokeCallback();
1696
1697  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
1698  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1699  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1700  respond(0, "200", false);
1701  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1702      Host::HealthFlag::FAILED_ACTIVE_HC));
1703  EXPECT_EQ(Host::Health::Unhealthy,
1704            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1705
1706  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1707  expectStreamCreate(0);
1708  test_sessions_[0]->interval_timer_->invokeCallback();
1709
1710  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
1711  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
1712  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1713  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1714  respond(0, "200", false);
1715  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1716}
1717
1718TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) {
1719  setupNoServiceValidationHCAlwaysLogFailure();
1720  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1721      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1722  expectSessionCreate();
1723  expectStreamCreate(0);
1724  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1725  health_checker_->start();
1726
1727  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
1728  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1729  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1730  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1731  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1732  respond(0, "503", false);
1733  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1734      Host::HealthFlag::FAILED_ACTIVE_HC));
1735  EXPECT_EQ(Host::Health::Unhealthy,
1736            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1737
1738  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
1739            Host::ActiveHealthFailureType::UNHEALTHY);
1740  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1741  expectStreamCreate(0);
1742  test_sessions_[0]->interval_timer_->invokeCallback();
1743
1744  // logUnhealthy is called with first_check == false
1745  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
1746  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1747  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1748  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, false));
1749  respond(0, "503", false);
1750  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1751      Host::HealthFlag::FAILED_ACTIVE_HC));
1752  EXPECT_EQ(Host::Health::Unhealthy,
1753            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1754
1755  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
1756            Host::ActiveHealthFailureType::UNHEALTHY);
1757  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1758  expectStreamCreate(0);
1759  test_sessions_[0]->interval_timer_->invokeCallback();
1760
1761  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
1762  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1763  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1764  respond(0, "200", false);
1765  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1766      Host::HealthFlag::FAILED_ACTIVE_HC));
1767  EXPECT_EQ(Host::Health::Unhealthy,
1768            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1769
1770  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1771  expectStreamCreate(0);
1772  test_sessions_[0]->interval_timer_->invokeCallback();
1773
1774  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
1775  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
1776  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1777  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1778  respond(0, "200", false);
1779  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1780}
1781
1782TEST_F(HttpHealthCheckerImplTest, Disconnect) {
1783  setupNoServiceValidationHC();
1784  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1785  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)).Times(1);
1786
1787  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1788      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1789  expectSessionCreate();
1790  expectStreamCreate(0);
1791  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1792  health_checker_->start();
1793
1794  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1795  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1796  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
1797  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1798
1799  expectClientCreate(0);
1800  expectStreamCreate(0);
1801  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1802  test_sessions_[0]->interval_timer_->invokeCallback();
1803
1804  EXPECT_CALL(*this, onHostStatus(cluster_->prioritySet().getMockHostSet(0)->hosts_[0],
1805                                  HealthTransition::Changed));
1806  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1807  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1808  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1809  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
1810  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1811      Host::HealthFlag::FAILED_ACTIVE_HC));
1812  EXPECT_EQ(Host::Health::Unhealthy,
1813            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1814}
1815
1816TEST_F(HttpHealthCheckerImplTest, Timeout) {
1817  setupNoServiceValidationHCOneUnhealthy();
1818  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1819      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1820  expectSessionCreate();
1821  expectStreamCreate(0);
1822  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1823  health_checker_->start();
1824
1825  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
1826  EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_));
1827  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1828  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1829  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1830  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1831  test_sessions_[0]->timeout_timer_->invokeCallback();
1832  EXPECT_EQ(Host::Health::Unhealthy,
1833            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1834
1835  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
1836            Host::ActiveHealthFailureType::TIMEOUT);
1837}
1838
1839// Make sure that a timeout during a partial response works correctly.
1840TEST_F(HttpHealthCheckerImplTest, TimeoutThenSuccess) {
1841  setupNoServiceValidationHC();
1842
1843  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1844      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1845  expectSessionCreate();
1846  expectStreamCreate(0);
1847  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1848  health_checker_->start();
1849
1850  // Do a response that is not complete but includes headers.
1851  std::unique_ptr<Http::TestResponseHeaderMapImpl> response_headers(
1852      new Http::TestResponseHeaderMapImpl{{":status", "200"}});
1853  test_sessions_[0]->stream_response_callbacks_->decodeHeaders(std::move(response_headers), false);
1854
1855  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
1856  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1857  EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_));
1858  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1859  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1860  test_sessions_[0]->timeout_timer_->invokeCallback();
1861  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1862
1863  expectClientCreate(0);
1864  expectStreamCreate(0);
1865  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1866  test_sessions_[0]->interval_timer_->invokeCallback();
1867
1868  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
1869  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1870  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1871  respond(0, "200", false, false, true);
1872  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1873}
1874
1875TEST_F(HttpHealthCheckerImplTest, TimeoutThenRemoteClose) {
1876  setupNoServiceValidationHC();
1877  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1878  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1879      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1880  expectSessionCreate();
1881  expectStreamCreate(0);
1882  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1883  health_checker_->start();
1884
1885  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
1886  EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_));
1887  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1888  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1889  test_sessions_[0]->timeout_timer_->invokeCallback();
1890  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1891
1892  expectClientCreate(0);
1893  expectStreamCreate(0);
1894  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1895  test_sessions_[0]->interval_timer_->invokeCallback();
1896
1897  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
1898  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1899  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1900  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1901  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
1902  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
1903      Host::HealthFlag::FAILED_ACTIVE_HC));
1904  EXPECT_EQ(Host::Health::Unhealthy,
1905            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1906
1907  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
1908            Host::ActiveHealthFailureType::TIMEOUT);
1909}
1910
1911TEST_F(HttpHealthCheckerImplTest, TimeoutAfterDisconnect) {
1912  setupNoServiceValidationHC();
1913  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1914      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1915  expectSessionCreate();
1916  expectStreamCreate(0);
1917  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
1918  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)).Times(2);
1919  health_checker_->start();
1920
1921  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)).Times(1);
1922  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)).Times(2);
1923  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1924  for (auto& session : test_sessions_) {
1925    session->client_connection_->close(Network::ConnectionCloseType::NoFlush);
1926  }
1927
1928  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)).Times(1);
1929  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
1930  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1931
1932  test_sessions_[0]->timeout_timer_->enableTimer(std::chrono::seconds(10), nullptr);
1933  test_sessions_[0]->timeout_timer_->invokeCallback();
1934  EXPECT_EQ(Host::Health::Unhealthy,
1935            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1936}
1937
1938TEST_F(HttpHealthCheckerImplTest, DynamicAddAndRemove) {
1939  setupNoServiceValidationHC();
1940  health_checker_->start();
1941
1942  expectSessionCreate();
1943  expectStreamCreate(0);
1944  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1945      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1946  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1947  cluster_->prioritySet().getMockHostSet(0)->runCallbacks(
1948      {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {});
1949
1950  HostVector removed{cluster_->prioritySet().getMockHostSet(0)->hosts_.back()};
1951  cluster_->prioritySet().getMockHostSet(0)->hosts_.clear();
1952  EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_));
1953  cluster_->prioritySet().getMockHostSet(0)->runCallbacks({}, removed);
1954}
1955
1956TEST_F(HttpHealthCheckerImplTest, ConnectionClose) {
1957  setupNoServiceValidationHC();
1958  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
1959
1960  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1961      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1962  expectSessionCreate();
1963  expectStreamCreate(0);
1964  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1965  health_checker_->start();
1966
1967  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1968  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1969  respond(0, "200", true);
1970  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1971
1972  expectClientCreate(0);
1973  expectStreamCreate(0);
1974  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1975  test_sessions_[0]->interval_timer_->invokeCallback();
1976}
1977
1978TEST_F(HttpHealthCheckerImplTest, ProxyConnectionClose) {
1979  setupNoServiceValidationHC();
1980  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
1981
1982  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
1983      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
1984  expectSessionCreate();
1985  expectStreamCreate(0);
1986  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1987  health_checker_->start();
1988
1989  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
1990  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
1991  respond(0, "200", false, true);
1992  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
1993
1994  expectClientCreate(0);
1995  expectStreamCreate(0);
1996  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
1997  test_sessions_[0]->interval_timer_->invokeCallback();
1998}
1999
2000TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) {
2001  setupHealthCheckIntervalOverridesHC();
2002  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2003      makeTestHost(cluster_->info_, "tcp://128.0.0.1:80")};
2004  expectSessionCreate();
2005  expectStreamCreate(0);
2006  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2007  health_checker_->start();
2008
2009  // First check should respect no_traffic_interval setting.
2010  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2011  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000), _));
2012  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2013  respond(0, "200", false);
2014  cluster_->info_->stats().upstream_cx_total_.inc();
2015
2016  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2017  // Needed after a response is sent.
2018  expectStreamCreate(0);
2019  test_sessions_[0]->interval_timer_->invokeCallback();
2020
2021  // Follow up successful checks should respect interval setting.
2022  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2023  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
2024  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2025  respond(0, "200", false);
2026
2027  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2028  // Needed after a response is sent.
2029  expectStreamCreate(0);
2030  test_sessions_[0]->interval_timer_->invokeCallback();
2031
2032  // Follow up successful checks should respect interval setting.
2033  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2034  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
2035  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2036  respond(0, "200", false);
2037
2038  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2039  // Needed after a response is sent.
2040  expectStreamCreate(0);
2041  test_sessions_[0]->interval_timer_->invokeCallback();
2042
2043  // A logical failure is not considered a network failure, therefore the unhealthy threshold is
2044  // ignored and health state changes immediately. Since the threshold is ignored, next health
2045  // check respects "unhealthy_interval".
2046  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
2047  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
2048  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
2049  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2050  respond(0, "503", false);
2051
2052  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2053  // Needed after a response is sent.
2054  expectStreamCreate(0);
2055  test_sessions_[0]->interval_timer_->invokeCallback();
2056
2057  // Subsequent failing checks should respect unhealthy_interval.
2058  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2059  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
2060  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2061  respond(0, "503", false);
2062
2063  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2064  // Needed after a response is sent.
2065  expectStreamCreate(0);
2066  test_sessions_[0]->interval_timer_->invokeCallback();
2067
2068  // Subsequent failing checks should respect unhealthy_interval.
2069  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2070  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
2071  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2072  respond(0, "503", false);
2073
2074  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2075  // Needed after a response is sent.
2076  expectStreamCreate(0);
2077  test_sessions_[0]->interval_timer_->invokeCallback();
2078
2079  // When transitioning to a successful state, checks should respect healthy_edge_interval. Health
2080  // state should be delayed pending healthy threshold.
2081  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
2082  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
2083  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2084  respond(0, "200", false);
2085
2086  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2087  // Needed after a response is sent.
2088  expectStreamCreate(0);
2089  test_sessions_[0]->interval_timer_->invokeCallback();
2090
2091  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
2092  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
2093  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2094  respond(0, "200", false);
2095
2096  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2097  // Needed after a response is sent.
2098  expectStreamCreate(0);
2099  test_sessions_[0]->interval_timer_->invokeCallback();
2100
2101  // After the healthy threshold is reached, health state should change while checks should respect
2102  // the default interval.
2103  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
2104  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
2105  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
2106  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2107  respond(0, "200", false);
2108
2109  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2110  // Needed after a response is sent.
2111  expectStreamCreate(0);
2112  test_sessions_[0]->interval_timer_->invokeCallback();
2113
2114  // Subsequent checks shouldn't change the state.
2115  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2116  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
2117  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2118  respond(0, "200", false);
2119
2120  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2121  // Needed after a response is sent.
2122  expectStreamCreate(0);
2123  test_sessions_[0]->interval_timer_->invokeCallback();
2124
2125  // First failed check after a run o successful ones should respect unhealthy_edge_interval. A
2126  // timeout, being a network type failure, should respect unhealthy threshold before changing the
2127  // health state.
2128  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
2129  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _));
2130  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2131  test_sessions_[0]->timeout_timer_->invokeCallback();
2132
2133  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2134  // Needed after a network timeout.
2135  expectClientCreate(0);
2136  // Needed after a response is sent.
2137  expectStreamCreate(0);
2138  test_sessions_[0]->interval_timer_->invokeCallback();
2139
2140  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
2141  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _));
2142  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2143  test_sessions_[0]->timeout_timer_->invokeCallback();
2144
2145  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2146  // Needed after a network timeout.
2147  expectClientCreate(0);
2148  // Needed after a response is sent.
2149  expectStreamCreate(0);
2150  test_sessions_[0]->interval_timer_->invokeCallback();
2151
2152  // Subsequent failing checks should respect unhealthy_interval. As the unhealthy threshold is
2153  // reached, health state should also change.
2154  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
2155  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
2156  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
2157  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2158  test_sessions_[0]->timeout_timer_->invokeCallback();
2159
2160  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2161  // Needed after a network timeout.
2162  expectClientCreate(0);
2163  // Needed after a response is sent.
2164  expectStreamCreate(0);
2165  test_sessions_[0]->interval_timer_->invokeCallback();
2166
2167  // Remaining failing checks shouldn't change the state.
2168  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2169  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
2170  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2171  test_sessions_[0]->timeout_timer_->invokeCallback();
2172
2173  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2174  // Needed after a network timeout.
2175  expectClientCreate(0);
2176  // Needed after a response is sent.
2177  expectStreamCreate(0);
2178  test_sessions_[0]->interval_timer_->invokeCallback();
2179
2180  // When transitioning to a successful state, checks should respect healthy_edge_interval.
2181  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
2182  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
2183  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2184  respond(0, "200", false);
2185
2186  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2187  // Needed after a response is sent.
2188  expectStreamCreate(0);
2189  test_sessions_[0]->interval_timer_->invokeCallback();
2190
2191  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
2192  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
2193  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2194  respond(0, "200", false);
2195
2196  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2197  // Needed after a response is sent.
2198  expectStreamCreate(0);
2199  test_sessions_[0]->interval_timer_->invokeCallback();
2200
2201  // After the healthy threshold is reached, health state should change while checks should respect
2202  // the default interval.
2203  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
2204  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
2205  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
2206  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2207  respond(0, "200", false);
2208
2209  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2210  // Needed after a response is sent.
2211  expectStreamCreate(0);
2212  test_sessions_[0]->interval_timer_->invokeCallback();
2213
2214  // Subsequent checks shouldn't change the state.
2215  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2216  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
2217  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2218  respond(0, "200", false);
2219}
2220
2221TEST_F(HttpHealthCheckerImplTest, RemoteCloseBetweenChecks) {
2222  setupNoServiceValidationHC();
2223  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(2);
2224
2225  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2226      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2227  expectSessionCreate();
2228  expectStreamCreate(0);
2229  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2230  health_checker_->start();
2231
2232  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
2233  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2234  respond(0, "200", false);
2235  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2236
2237  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
2238
2239  expectClientCreate(0);
2240  expectStreamCreate(0);
2241  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2242  test_sessions_[0]->interval_timer_->invokeCallback();
2243
2244  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
2245  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2246  respond(0, "200", false);
2247  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2248}
2249
2250// Test that we close connections on a healthy check when reuse_connection is false.
2251TEST_F(HttpHealthCheckerImplTest, DontReuseConnectionBetweenChecks) {
2252  setupNoServiceValidationNoReuseConnectionHC();
2253  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(2);
2254
2255  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2256      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2257  expectSessionCreate();
2258  expectStreamCreate(0);
2259  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2260  health_checker_->start();
2261
2262  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
2263  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2264  respond(0, "200", false);
2265  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2266
2267  // A new client is created because we close the connection ourselves.
2268  // See HttpHealthCheckerImplTest.RemoteCloseBetweenChecks for how this works when the remote end
2269  // closes the connection.
2270  expectClientCreate(0);
2271  expectStreamCreate(0);
2272  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2273  test_sessions_[0]->interval_timer_->invokeCallback();
2274
2275  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
2276  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2277  respond(0, "200", false);
2278  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2279}
2280
2281TEST_F(HttpHealthCheckerImplTest, StreamReachesWatermarkDuringCheck) {
2282  setupNoServiceValidationHC();
2283  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2284
2285  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2286      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2287  expectSessionCreate();
2288  expectStreamCreate(0);
2289  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2290  health_checker_->start();
2291
2292  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
2293  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2294
2295  test_sessions_[0]->request_encoder_.stream_.runHighWatermarkCallbacks();
2296  test_sessions_[0]->request_encoder_.stream_.runLowWatermarkCallbacks();
2297
2298  respond(0, "200", true);
2299  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2300}
2301
2302TEST_F(HttpHealthCheckerImplTest, ConnectionReachesWatermarkDuringCheck) {
2303  setupNoServiceValidationHC();
2304  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
2305
2306  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2307      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2308  expectSessionCreate();
2309  expectStreamCreate(0);
2310  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2311  health_checker_->start();
2312
2313  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _));
2314  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2315
2316  test_sessions_[0]->client_connection_->runHighWatermarkCallbacks();
2317  test_sessions_[0]->client_connection_->runLowWatermarkCallbacks();
2318
2319  respond(0, "200", true);
2320  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2321}
2322
2323TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithAltPort) {
2324  const std::string host = "fake_cluster";
2325  const std::string path = "/healthcheck";
2326  setupServiceValidationHC();
2327  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
2328      .WillOnce(Return(true));
2329
2330  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
2331
2332  // Prepares a host with its designated health check port.
2333  const HostWithHealthCheckMap hosts{{"127.0.0.1:80", makeHealthCheckConfig(8000)}};
2334  appendTestHosts(cluster_, hosts);
2335  cluster_->info_->stats().upstream_cx_total_.inc();
2336  expectSessionCreate(hosts);
2337  expectStreamCreate(0);
2338  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2339  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
2340      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
2341        EXPECT_EQ(headers.Host()->value().getStringView(), host);
2342        EXPECT_EQ(headers.Path()->value().getStringView(), path);
2343      }));
2344  health_checker_->start();
2345
2346  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
2347  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
2348      .WillOnce(Return(45000));
2349  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
2350              enableTimer(std::chrono::milliseconds(45000), _));
2351  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2352  absl::optional<std::string> health_checked_cluster("locations-production-iad");
2353  respond(0, "200", false, false, true, false, health_checked_cluster);
2354  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2355}
2356
2357// Test host check success with multiple hosts by checking each host defined health check port.
2358TEST_F(HttpHealthCheckerImplTest, SuccessWithMultipleHostsAndAltPort) {
2359  setupNoServiceValidationHC();
2360  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(2);
2361
2362  // Prepares a set of hosts along with its designated health check ports.
2363  const HostWithHealthCheckMap hosts = {{"127.0.0.1:80", makeHealthCheckConfig(8000)},
2364                                        {"127.0.0.1:81", makeHealthCheckConfig(8001)}};
2365  appendTestHosts(cluster_, hosts);
2366  cluster_->info_->stats().upstream_cx_total_.inc();
2367  cluster_->info_->stats().upstream_cx_total_.inc();
2368  expectSessionCreate(hosts);
2369  expectStreamCreate(0);
2370  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2371  expectSessionCreate(hosts);
2372  expectStreamCreate(1);
2373  EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_, _));
2374  health_checker_->start();
2375
2376  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).Times(2);
2377  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
2378      .Times(2)
2379      .WillRepeatedly(Return(45000));
2380  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
2381              enableTimer(std::chrono::milliseconds(45000), _));
2382  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2383  EXPECT_CALL(*test_sessions_[1]->interval_timer_,
2384              enableTimer(std::chrono::milliseconds(45000), _));
2385  EXPECT_CALL(*test_sessions_[1]->timeout_timer_, disableTimer());
2386  respond(0, "200", false, false, true);
2387  respond(1, "200", false, false, true);
2388  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2389  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[1]->health());
2390}
2391
2392TEST_F(HttpHealthCheckerImplTest, Http2ClusterUseHttp2CodecClient) {
2393  setupNoServiceValidationHCWithHttp2();
2394  EXPECT_EQ(Http::CodecClient::Type::HTTP2, health_checker_->codecClientType());
2395}
2396
2397MATCHER_P(MetadataEq, expected, "") {
2398  const envoy::config::core::v3::Metadata* metadata = arg;
2399  if (!metadata) {
2400    return false;
2401  }
2402  EXPECT_TRUE(Envoy::Protobuf::util::MessageDifferencer::Equals(*metadata, expected));
2403  return true;
2404}
2405
2406TEST_F(HttpHealthCheckerImplTest, TransportSocketMatchCriteria) {
2407  const std::string host = "fake_cluster";
2408  const std::string path = "/healthcheck";
2409  const std::string yaml = R"EOF(
2410    timeout: 1s
2411    interval: 1s
2412    no_traffic_interval: 1s
2413    interval_jitter_percent: 40
2414    unhealthy_threshold: 2
2415    healthy_threshold: 2
2416    http_health_check:
2417      service_name_matcher:
2418        prefix: locations
2419      path: /healthcheck
2420    transport_socket_match_criteria:
2421      key: value
2422    )EOF";
2423
2424  auto default_socket_factory = std::make_unique<Network::MockTransportSocketFactory>();
2425  // We expect that this default_socket_factory will NOT be used to create a transport socket for
2426  // the health check connection.
2427  EXPECT_CALL(*default_socket_factory, createTransportSocket(_)).Times(0);
2428  EXPECT_CALL(*default_socket_factory, implementsSecureTransport());
2429  auto transport_socket_match =
2430      std::make_unique<Upstream::MockTransportSocketMatcher>(std::move(default_socket_factory));
2431
2432  auto metadata = TestUtility::parseYaml<envoy::config::core::v3::Metadata>(
2433      R"EOF(
2434    filter_metadata:
2435      envoy.transport_socket_match:
2436        key: value
2437  )EOF");
2438
2439  Stats::IsolatedStoreImpl stats_store;
2440  auto health_transport_socket_stats = TransportSocketMatchStats{
2441      ALL_TRANSPORT_SOCKET_MATCH_STATS(POOL_COUNTER_PREFIX(stats_store, "test"))};
2442  auto health_check_only_socket_factory = std::make_unique<Network::MockTransportSocketFactory>();
2443
2444  // We expect resolve() to be called twice, once for endpoint socket matching (with no metadata in
2445  // this test) and once for health check socket matching. In the latter we expect metadata that
2446  // matches the above object.
2447  EXPECT_CALL(*transport_socket_match, resolve(nullptr));
2448  EXPECT_CALL(*transport_socket_match, resolve(MetadataEq(metadata)))
2449      .WillOnce(Return(TransportSocketMatcher::MatchData(
2450          *health_check_only_socket_factory, health_transport_socket_stats, "health_check_only")));
2451  // The health_check_only_socket_factory should be used to create a transport socket for the health
2452  // check connection.
2453  EXPECT_CALL(*health_check_only_socket_factory, createTransportSocket(_));
2454
2455  cluster_->info_->transport_socket_matcher_ = std::move(transport_socket_match);
2456
2457  allocHealthChecker(yaml);
2458
2459  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2460      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2461  cluster_->info_->stats().upstream_cx_total_.inc();
2462  expectSessionCreate();
2463  expectStreamCreate(0);
2464  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2465  health_checker_->start();
2466  EXPECT_EQ(health_transport_socket_stats.total_match_count_.value(), 1);
2467}
2468
2469TEST_F(HttpHealthCheckerImplTest, NoTransportSocketMatchCriteria) {
2470  const std::string host = "fake_cluster";
2471  const std::string path = "/healthcheck";
2472  const std::string yaml = R"EOF(
2473    timeout: 1s
2474    interval: 1s
2475    no_traffic_interval: 1s
2476    interval_jitter_percent: 40
2477    unhealthy_threshold: 2
2478    healthy_threshold: 2
2479    http_health_check:
2480      service_name_matcher:
2481        prefix: locations
2482      path: /healthcheck
2483    )EOF";
2484
2485  auto default_socket_factory = std::make_unique<Network::MockTransportSocketFactory>();
2486  // The default_socket_factory should be used to create a transport socket for the health check
2487  // connection.
2488  EXPECT_CALL(*default_socket_factory, createTransportSocket(_));
2489  EXPECT_CALL(*default_socket_factory, implementsSecureTransport());
2490  auto transport_socket_match =
2491      std::make_unique<Upstream::MockTransportSocketMatcher>(std::move(default_socket_factory));
2492  // We expect resolve() to be called exactly once for endpoint socket matching. We should not
2493  // attempt to match again for health checks since there is not match criteria in the config.
2494  EXPECT_CALL(*transport_socket_match, resolve(nullptr));
2495
2496  cluster_->info_->transport_socket_matcher_ = std::move(transport_socket_match);
2497
2498  allocHealthChecker(yaml);
2499
2500  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2501      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2502  cluster_->info_->stats().upstream_cx_total_.inc();
2503  expectSessionCreate();
2504  expectStreamCreate(0);
2505  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2506  health_checker_->start();
2507}
2508
2509class TestProdHttpHealthChecker : public ProdHttpHealthCheckerImpl {
2510public:
2511  using ProdHttpHealthCheckerImpl::ProdHttpHealthCheckerImpl;
2512
2513  std::unique_ptr<Http::CodecClient>
2514  createCodecClientForTest(std::unique_ptr<Network::ClientConnection>&& connection) {
2515    Upstream::Host::CreateConnectionData data;
2516    data.connection_ = std::move(connection);
2517    data.host_description_ = std::make_shared<NiceMock<Upstream::MockHostDescription>>();
2518    return std::unique_ptr<Http::CodecClient>(createCodecClient(data));
2519  }
2520};
2521
2522class ProdHttpHealthCheckerTest : public testing::Test, public HealthCheckerTestBase {
2523public:
2524  void allocHealthChecker(const std::string& yaml) {
2525    health_checker_ = std::make_shared<TestProdHttpHealthChecker>(
2526        *cluster_, parseHealthCheckFromV2Yaml(yaml), dispatcher_, runtime_, random_,
2527        HealthCheckEventLoggerPtr(event_logger_));
2528  }
2529
2530  void addCompletionCallback() {
2531    health_checker_->addHostCheckCompleteCb(
2532        [this](HostSharedPtr host, HealthTransition changed_state) -> void {
2533          onHostStatus(host, changed_state);
2534        });
2535  }
2536
2537  void setupNoServiceValidationHCWithHttp2() {
2538    const std::string yaml = R"EOF(
2539    timeout: 1s
2540    interval: 1s
2541    no_traffic_interval: 5s
2542    interval_jitter: 1s
2543    unhealthy_threshold: 2
2544    healthy_threshold: 2
2545    http_health_check:
2546      service_name_matcher:
2547        prefix: locations
2548      path: /healthcheck
2549      codec_client_type: Http2
2550    )EOF";
2551
2552    allocHealthChecker(yaml);
2553    addCompletionCallback();
2554  }
2555
2556  void setupNoServiceValidationHC() {
2557    const std::string yaml = R"EOF(
2558    timeout: 1s
2559    interval: 1s
2560    no_traffic_interval: 5s
2561    interval_jitter: 1s
2562    unhealthy_threshold: 2
2563    healthy_threshold: 2
2564    http_health_check:
2565      service_name_matcher:
2566        prefix: locations
2567      path: /healthcheck
2568    )EOF";
2569
2570    allocHealthChecker(yaml);
2571    addCompletionCallback();
2572  }
2573
2574  MOCK_METHOD(void, onHostStatus, (HostSharedPtr host, HealthTransition changed_state));
2575  std::unique_ptr<Network::MockClientConnection> connection_ =
2576      std::make_unique<NiceMock<Network::MockClientConnection>>();
2577  std::shared_ptr<TestProdHttpHealthChecker> health_checker_;
2578};
2579
2580TEST_F(ProdHttpHealthCheckerTest, ProdHttpHealthCheckerH1HealthChecking) {
2581  setupNoServiceValidationHC();
2582  EXPECT_EQ(Http::CodecClient::Type::HTTP1,
2583            health_checker_->createCodecClientForTest(std::move(connection_))->type());
2584}
2585
2586TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(Http1CodecClient)) {
2587  const std::string yaml = R"EOF(
2588    timeout: 1s
2589    interval: 1s
2590    no_traffic_interval: 5s
2591    interval_jitter: 1s
2592    unhealthy_threshold: 2
2593    healthy_threshold: 2
2594    http_health_check:
2595      service_name_matcher:
2596        prefix: locations
2597      path: /healthcheck
2598      use_http2: false
2599    )EOF";
2600
2601  allocHealthChecker(yaml);
2602  addCompletionCallback();
2603  EXPECT_EQ(Http::CodecClient::Type::HTTP1, health_checker_->codecClientType());
2604}
2605
2606TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(Http2CodecClient)) {
2607  const std::string yaml = R"EOF(
2608    timeout: 1s
2609    interval: 1s
2610    no_traffic_interval: 5s
2611    interval_jitter: 1s
2612    unhealthy_threshold: 2
2613    healthy_threshold: 2
2614    http_health_check:
2615      service_name_matcher:
2616        prefix: locations
2617      path: /healthcheck
2618      use_http2: true
2619    )EOF";
2620
2621  allocHealthChecker(yaml);
2622  addCompletionCallback();
2623  EXPECT_EQ(Http::CodecClient::Type::HTTP2, health_checker_->codecClientType());
2624}
2625
2626TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(ServiceNameMatch)) {
2627  const std::string host = "fake_cluster";
2628  const std::string path = "/healthcheck";
2629  setupDeprecatedServiceNameValidationHC("locations");
2630  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
2631      .WillOnce(Return(true));
2632
2633  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)).Times(1);
2634
2635  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2636      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2637  cluster_->info_->stats().upstream_cx_total_.inc();
2638  expectSessionCreate();
2639  expectStreamCreate(0);
2640  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2641  EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true))
2642      .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
2643        EXPECT_EQ(headers.Host()->value().getStringView(), host);
2644        EXPECT_EQ(headers.Path()->value().getStringView(), path);
2645        EXPECT_EQ(headers.Scheme()->value().getStringView(),
2646                  Http::Headers::get().SchemeValues.Http);
2647      }));
2648  health_checker_->start();
2649
2650  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
2651  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
2652      .WillOnce(Return(45000));
2653  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
2654              enableTimer(std::chrono::milliseconds(45000), _));
2655  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2656  absl::optional<std::string> health_checked_cluster("locations-production-iad");
2657  respond(0, "200", false, false, true, false, health_checked_cluster);
2658  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2659}
2660
2661TEST_F(HttpHealthCheckerImplTest, DEPRECATED_FEATURE_TEST(ServiceNameMismatch)) {
2662  setupDeprecatedServiceNameValidationHC("locations");
2663  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
2664  EXPECT_CALL(runtime_.snapshot_, featureEnabled("health_check.verify_cluster", 100))
2665      .WillOnce(Return(true));
2666
2667  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)).Times(1);
2668  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
2669
2670  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
2671      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
2672  cluster_->info_->stats().upstream_cx_total_.inc();
2673  expectSessionCreate();
2674  expectStreamCreate(0);
2675  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
2676  health_checker_->start();
2677
2678  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
2679  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
2680      .WillOnce(Return(45000));
2681  EXPECT_CALL(*test_sessions_[0]->interval_timer_,
2682              enableTimer(std::chrono::milliseconds(45000), _));
2683  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
2684  absl::optional<std::string> health_checked_cluster("api-production-iad");
2685  respond(0, "200", false, false, true, false, health_checked_cluster);
2686  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
2687      Host::HealthFlag::FAILED_ACTIVE_HC));
2688  EXPECT_EQ(Host::Health::Unhealthy,
2689            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
2690}
2691
2692TEST_F(ProdHttpHealthCheckerTest, ProdHttpHealthCheckerH2HealthChecking) {
2693  setupNoServiceValidationHCWithHttp2();
2694  EXPECT_EQ(Http::CodecClient::Type::HTTP2,
2695            health_checker_->createCodecClientForTest(std::move(connection_))->type());
2696}
2697
2698TEST(HttpStatusChecker, Default) {
2699  const std::string yaml = R"EOF(
2700  timeout: 1s
2701  interval: 1s
2702  unhealthy_threshold: 2
2703  healthy_threshold: 2
2704  http_health_check:
2705    service_name_matcher:
2706        prefix: locations
2707    path: /healthcheck
2708  )EOF";
2709
2710  HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2711      parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200);
2712
2713  EXPECT_TRUE(http_status_checker.inRange(200));
2714  EXPECT_FALSE(http_status_checker.inRange(204));
2715}
2716
2717TEST(HttpStatusChecker, Single100) {
2718  const std::string yaml = R"EOF(
2719  timeout: 1s
2720  interval: 1s
2721  unhealthy_threshold: 2
2722  healthy_threshold: 2
2723  http_health_check:
2724    service_name_matcher:
2725        prefix: locations
2726    path: /healthcheck
2727    expected_statuses:
2728      - start: 100
2729        end: 101
2730  )EOF";
2731
2732  HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2733      parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200);
2734
2735  EXPECT_FALSE(http_status_checker.inRange(200));
2736
2737  EXPECT_FALSE(http_status_checker.inRange(99));
2738  EXPECT_TRUE(http_status_checker.inRange(100));
2739  EXPECT_FALSE(http_status_checker.inRange(101));
2740}
2741
2742TEST(HttpStatusChecker, Single599) {
2743  const std::string yaml = R"EOF(
2744  timeout: 1s
2745  interval: 1s
2746  unhealthy_threshold: 2
2747  healthy_threshold: 2
2748  http_health_check:
2749    service_name_matcher:
2750        prefix: locations
2751    path: /healthcheck
2752    expected_statuses:
2753      - start: 599
2754        end: 600
2755  )EOF";
2756
2757  HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2758      parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200);
2759
2760  EXPECT_FALSE(http_status_checker.inRange(200));
2761
2762  EXPECT_FALSE(http_status_checker.inRange(598));
2763  EXPECT_TRUE(http_status_checker.inRange(599));
2764  EXPECT_FALSE(http_status_checker.inRange(600));
2765}
2766
2767TEST(HttpStatusChecker, Ranges_204_304) {
2768  const std::string yaml = R"EOF(
2769  timeout: 1s
2770  interval: 1s
2771  unhealthy_threshold: 2
2772  healthy_threshold: 2
2773  http_health_check:
2774    service_name_matcher:
2775        prefix: locations
2776    path: /healthcheck
2777    expected_statuses:
2778      - start: 204
2779        end: 205
2780      - start: 304
2781        end: 305
2782  )EOF";
2783
2784  HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2785      parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200);
2786
2787  EXPECT_FALSE(http_status_checker.inRange(200));
2788
2789  EXPECT_FALSE(http_status_checker.inRange(203));
2790  EXPECT_TRUE(http_status_checker.inRange(204));
2791  EXPECT_FALSE(http_status_checker.inRange(205));
2792  EXPECT_FALSE(http_status_checker.inRange(303));
2793  EXPECT_TRUE(http_status_checker.inRange(304));
2794  EXPECT_FALSE(http_status_checker.inRange(305));
2795}
2796
2797TEST(HttpStatusChecker, Below100) {
2798  const std::string yaml = R"EOF(
2799  timeout: 1s
2800  interval: 1s
2801  unhealthy_threshold: 2
2802  healthy_threshold: 2
2803  http_health_check:
2804    service_name_matcher:
2805        prefix: locations
2806    path: /healthcheck
2807    expected_statuses:
2808      - start: 99
2809        end: 100
2810  )EOF";
2811
2812  EXPECT_THROW_WITH_MESSAGE(
2813      HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2814          parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200),
2815      EnvoyException, "Invalid http status range: expecting start >= 100, but found start=99");
2816}
2817
2818TEST(HttpStatusChecker, Above599) {
2819  const std::string yaml = R"EOF(
2820  timeout: 1s
2821  interval: 1s
2822  unhealthy_threshold: 2
2823  healthy_threshold: 2
2824  http_health_check:
2825    service_name_matcher:
2826        prefix: locations
2827    path: /healthchecka
2828    expected_statuses:
2829      - start: 600
2830        end: 601
2831  )EOF";
2832
2833  EXPECT_THROW_WITH_MESSAGE(
2834      HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2835          parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200),
2836      EnvoyException, "Invalid http status range: expecting end <= 600, but found end=601");
2837}
2838
2839TEST(HttpStatusChecker, InvalidRange) {
2840  const std::string yaml = R"EOF(
2841  timeout: 1s
2842  interval: 1s
2843  unhealthy_threshold: 2
2844  healthy_threshold: 2
2845  http_health_check:
2846    service_name_matcher:
2847        prefix: locations
2848    path: /healthchecka
2849    expected_statuses:
2850      - start: 200
2851        end: 200
2852  )EOF";
2853
2854  EXPECT_THROW_WITH_MESSAGE(
2855      HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2856          parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200),
2857      EnvoyException,
2858      "Invalid http status range: expecting start < end, but found start=200 and end=200");
2859}
2860
2861TEST(HttpStatusChecker, InvalidRange2) {
2862  const std::string yaml = R"EOF(
2863  timeout: 1s
2864  interval: 1s
2865  unhealthy_threshold: 2
2866  healthy_threshold: 2
2867  http_health_check:
2868    service_name_matcher:
2869        prefix: locations
2870    path: /healthchecka
2871    expected_statuses:
2872      - start: 201
2873        end: 200
2874  )EOF";
2875
2876  EXPECT_THROW_WITH_MESSAGE(
2877      HttpHealthCheckerImpl::HttpStatusChecker http_status_checker(
2878          parseHealthCheckFromV2Yaml(yaml).http_health_check().expected_statuses(), 200),
2879      EnvoyException,
2880      "Invalid http status range: expecting start < end, but found start=201 and end=200");
2881}
2882
2883TEST(TcpHealthCheckMatcher, loadJsonBytes) {
2884  {
2885    Protobuf::RepeatedPtrField<envoy::config::core::v3::HealthCheck::Payload> repeated_payload;
2886    repeated_payload.Add()->set_text("39000000");
2887    repeated_payload.Add()->set_text("EEEEEEEE");
2888
2889    TcpHealthCheckMatcher::MatchSegments segments =
2890        TcpHealthCheckMatcher::loadProtoBytes(repeated_payload);
2891    EXPECT_EQ(2U, segments.size());
2892  }
2893
2894  {
2895    Protobuf::RepeatedPtrField<envoy::config::core::v3::HealthCheck::Payload> repeated_payload;
2896    repeated_payload.Add()->set_text("4");
2897
2898    EXPECT_THROW(TcpHealthCheckMatcher::loadProtoBytes(repeated_payload), EnvoyException);
2899  }
2900
2901  {
2902    Protobuf::RepeatedPtrField<envoy::config::core::v3::HealthCheck::Payload> repeated_payload;
2903    repeated_payload.Add()->set_text("gg");
2904
2905    EXPECT_THROW(TcpHealthCheckMatcher::loadProtoBytes(repeated_payload), EnvoyException);
2906  }
2907}
2908
2909static void add_uint8(Buffer::Instance& buffer, uint8_t addend) {
2910  buffer.add(&addend, sizeof(addend));
2911}
2912
2913TEST(TcpHealthCheckMatcher, match) {
2914  Protobuf::RepeatedPtrField<envoy::config::core::v3::HealthCheck::Payload> repeated_payload;
2915  repeated_payload.Add()->set_text("01");
2916  repeated_payload.Add()->set_text("02");
2917
2918  TcpHealthCheckMatcher::MatchSegments segments =
2919      TcpHealthCheckMatcher::loadProtoBytes(repeated_payload);
2920
2921  Buffer::OwnedImpl buffer;
2922  EXPECT_FALSE(TcpHealthCheckMatcher::match(segments, buffer));
2923  add_uint8(buffer, 1);
2924  EXPECT_FALSE(TcpHealthCheckMatcher::match(segments, buffer));
2925  add_uint8(buffer, 2);
2926  EXPECT_TRUE(TcpHealthCheckMatcher::match(segments, buffer));
2927
2928  buffer.drain(2);
2929  add_uint8(buffer, 1);
2930  add_uint8(buffer, 3);
2931  add_uint8(buffer, 2);
2932  EXPECT_TRUE(TcpHealthCheckMatcher::match(segments, buffer));
2933
2934  buffer.drain(3);
2935  add_uint8(buffer, 0);
2936  add_uint8(buffer, 3);
2937  add_uint8(buffer, 1);
2938  add_uint8(buffer, 2);
2939  EXPECT_TRUE(TcpHealthCheckMatcher::match(segments, buffer));
2940}
2941
2942class TcpHealthCheckerImplTest : public testing::Test, public HealthCheckerTestBase {
2943public:
2944  void allocHealthChecker(const std::string& yaml) {
2945    health_checker_ = std::make_shared<TcpHealthCheckerImpl>(
2946        *cluster_, parseHealthCheckFromV2Yaml(yaml), dispatcher_, runtime_, random_,
2947        HealthCheckEventLoggerPtr(event_logger_));
2948  }
2949
2950  void setupData(unsigned int unhealthy_threshold = 2) {
2951    std::ostringstream yaml;
2952    yaml << R"EOF(
2953    timeout: 1s
2954    interval: 1s
2955    unhealthy_threshold: )EOF"
2956         << unhealthy_threshold << R"EOF(
2957    healthy_threshold: 2
2958    tcp_health_check:
2959      send:
2960        text: "01"
2961      receive:
2962      - text: "02"
2963    )EOF";
2964
2965    allocHealthChecker(yaml.str());
2966  }
2967
2968  void setupNoData() {
2969    std::string yaml = R"EOF(
2970    timeout: 1s
2971    interval: 1s
2972    unhealthy_threshold: 2
2973    healthy_threshold: 2
2974    tcp_health_check: {}
2975    )EOF";
2976
2977    allocHealthChecker(yaml);
2978  }
2979
2980  void setupDataDontReuseConnection() {
2981    std::string yaml = R"EOF(
2982    timeout: 1s
2983    interval: 1s
2984    unhealthy_threshold: 2
2985    healthy_threshold: 2
2986    reuse_connection: false
2987    tcp_health_check:
2988      send:
2989        text: "01"
2990      receive:
2991      - text: "02"
2992    )EOF";
2993
2994    allocHealthChecker(yaml);
2995  }
2996
2997  void expectSessionCreate() {
2998    interval_timer_ = new Event::MockTimer(&dispatcher_);
2999    timeout_timer_ = new Event::MockTimer(&dispatcher_);
3000  }
3001
3002  void expectClientCreate() {
3003    connection_ = new NiceMock<Network::MockClientConnection>();
3004    EXPECT_CALL(dispatcher_, createClientConnection_(_, _, _, _)).WillOnce(Return(connection_));
3005    EXPECT_CALL(*connection_, addReadFilter(_)).WillOnce(SaveArg<0>(&read_filter_));
3006  }
3007
3008  std::shared_ptr<TcpHealthCheckerImpl> health_checker_;
3009  Network::MockClientConnection* connection_{};
3010  Event::MockTimer* timeout_timer_{};
3011  Event::MockTimer* interval_timer_{};
3012  Network::ReadFilterSharedPtr read_filter_;
3013};
3014
3015TEST_F(TcpHealthCheckerImplTest, Success) {
3016  InSequence s;
3017
3018  setupData();
3019  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3020      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3021  expectSessionCreate();
3022  expectClientCreate();
3023  EXPECT_CALL(*connection_, write(_, _));
3024  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3025  health_checker_->start();
3026
3027  connection_->runHighWatermarkCallbacks();
3028  connection_->runLowWatermarkCallbacks();
3029  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3030
3031  EXPECT_CALL(*timeout_timer_, disableTimer());
3032  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3033  Buffer::OwnedImpl response;
3034  add_uint8(response, 2);
3035  read_filter_->onData(response, false);
3036}
3037
3038// Tests that a successful healthcheck will disconnect the client when reuse_connection is false.
3039TEST_F(TcpHealthCheckerImplTest, DataWithoutReusingConnection) {
3040  InSequence s;
3041
3042  setupDataDontReuseConnection();
3043  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3044      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3045  expectSessionCreate();
3046  expectClientCreate();
3047  EXPECT_CALL(*connection_, write(_, _)).Times(1);
3048  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3049  health_checker_->start();
3050
3051  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3052
3053  // Expected execution flow when a healthcheck is successful and reuse_connection is false.
3054  EXPECT_CALL(*timeout_timer_, disableTimer());
3055  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3056  EXPECT_CALL(*connection_, close(Network::ConnectionCloseType::NoFlush)).Times(1);
3057
3058  Buffer::OwnedImpl response;
3059  add_uint8(response, 2);
3060  read_filter_->onData(response, false);
3061
3062  // These are the expected metric results after testing.
3063  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3064  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3065}
3066
3067// Tests an unsuccessful healthcheck, where the endpoint sends wrong data
3068TEST_F(TcpHealthCheckerImplTest, WrongData) {
3069  InSequence s;
3070
3071  setupDataDontReuseConnection();
3072  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3073      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3074  expectSessionCreate();
3075  expectClientCreate();
3076  EXPECT_CALL(*connection_, write(_, _)).Times(1);
3077  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3078  health_checker_->start();
3079
3080  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3081
3082  // Not the expected response
3083  Buffer::OwnedImpl response;
3084  add_uint8(response, 3);
3085  read_filter_->onData(response, false);
3086
3087  // These are the expected metric results after testing.
3088  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3089  // TODO(lilika): This should indicate a failure
3090  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3091  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
3092            Host::ActiveHealthFailureType::UNHEALTHY);
3093}
3094
3095TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) {
3096  InSequence s;
3097
3098  setupData();
3099  health_checker_->start();
3100
3101  expectSessionCreate();
3102  expectClientCreate();
3103  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3104      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3105  EXPECT_CALL(*connection_, write(_, _));
3106  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3107
3108  cluster_->prioritySet().getMockHostSet(0)->runCallbacks(
3109      {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {});
3110
3111  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3112
3113  Buffer::OwnedImpl response;
3114  add_uint8(response, 1);
3115  read_filter_->onData(response, false);
3116
3117  EXPECT_CALL(*connection_, close(_));
3118  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
3119  EXPECT_CALL(*timeout_timer_, disableTimer());
3120  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3121  timeout_timer_->invokeCallback();
3122  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
3123            Host::ActiveHealthFailureType::TIMEOUT);
3124  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3125
3126  expectClientCreate();
3127  EXPECT_CALL(*connection_, write(_, _));
3128  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3129  interval_timer_->invokeCallback();
3130
3131  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3132
3133  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
3134  EXPECT_CALL(*timeout_timer_, disableTimer());
3135  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3136  connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
3137  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
3138      Host::HealthFlag::FAILED_ACTIVE_HC));
3139  EXPECT_EQ(Host::Health::Unhealthy,
3140            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3141
3142  expectClientCreate();
3143  EXPECT_CALL(*connection_, write(_, _));
3144  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3145  interval_timer_->invokeCallback();
3146
3147  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3148
3149  HostVector removed{cluster_->prioritySet().getMockHostSet(0)->hosts_.back()};
3150  cluster_->prioritySet().getMockHostSet(0)->hosts_.clear();
3151  EXPECT_CALL(*connection_, close(_));
3152  cluster_->prioritySet().getMockHostSet(0)->runCallbacks({}, removed);
3153}
3154
3155TEST_F(TcpHealthCheckerImplTest, Timeout) {
3156  InSequence s;
3157
3158  setupData(1);
3159  health_checker_->start();
3160
3161  expectSessionCreate();
3162  expectClientCreate();
3163  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3164      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3165  EXPECT_CALL(*connection_, write(_, _));
3166  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3167
3168  cluster_->prioritySet().getMockHostSet(0)->runCallbacks(
3169      {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {});
3170
3171  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3172
3173  Buffer::OwnedImpl response;
3174  add_uint8(response, 1);
3175  read_filter_->onData(response, false);
3176
3177  EXPECT_CALL(*connection_, close(_));
3178  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
3179  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
3180  EXPECT_CALL(*timeout_timer_, disableTimer());
3181  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3182  timeout_timer_->invokeCallback();
3183  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
3184            Host::ActiveHealthFailureType::TIMEOUT);
3185  EXPECT_EQ(Host::Health::Unhealthy,
3186            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3187}
3188
3189TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) {
3190  InSequence s;
3191
3192  setupData();
3193  health_checker_->start();
3194
3195  expectSessionCreate();
3196  expectClientCreate();
3197  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3198      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3199  EXPECT_CALL(*connection_, write(_, _));
3200  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3201
3202  cluster_->prioritySet().getMockHostSet(0)->runCallbacks(
3203      {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {});
3204
3205  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3206
3207  Buffer::OwnedImpl response;
3208  add_uint8(response, 1);
3209  read_filter_->onData(response, false);
3210
3211  EXPECT_CALL(*connection_, close(_));
3212  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
3213  EXPECT_CALL(*timeout_timer_, disableTimer());
3214  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3215  timeout_timer_->invokeCallback();
3216  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
3217            Host::ActiveHealthFailureType::TIMEOUT);
3218  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3219
3220  expectClientCreate();
3221  EXPECT_CALL(*connection_, write(_, _));
3222  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3223  interval_timer_->invokeCallback();
3224
3225  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3226
3227  EXPECT_CALL(*connection_, close(_));
3228  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
3229  EXPECT_CALL(*timeout_timer_, disableTimer());
3230  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3231  timeout_timer_->invokeCallback();
3232  EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(),
3233            Host::ActiveHealthFailureType::TIMEOUT);
3234  EXPECT_EQ(Host::Health::Unhealthy,
3235            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3236
3237  expectClientCreate();
3238  EXPECT_CALL(*connection_, write(_, _));
3239  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3240  interval_timer_->invokeCallback();
3241
3242  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3243
3244  HostVector removed{cluster_->prioritySet().getMockHostSet(0)->hosts_.back()};
3245  cluster_->prioritySet().getMockHostSet(0)->hosts_.clear();
3246  EXPECT_CALL(*connection_, close(_));
3247  cluster_->prioritySet().getMockHostSet(0)->runCallbacks({}, removed);
3248}
3249
3250// Tests that when reuse_connection is false timeouts execute normally.
3251TEST_F(TcpHealthCheckerImplTest, TimeoutWithoutReusingConnection) {
3252  InSequence s;
3253
3254  setupDataDontReuseConnection();
3255  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3256      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3257  expectSessionCreate();
3258  expectClientCreate();
3259  EXPECT_CALL(*connection_, write(_, _)).Times(1);
3260  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3261  health_checker_->start();
3262
3263  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3264
3265  // Expected flow when a healthcheck is successful and reuse_connection is false.
3266  EXPECT_CALL(*timeout_timer_, disableTimer());
3267  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3268  EXPECT_CALL(*connection_, close(Network::ConnectionCloseType::NoFlush)).Times(1);
3269
3270  Buffer::OwnedImpl response;
3271  add_uint8(response, 2);
3272  read_filter_->onData(response, false);
3273
3274  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3275  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3276
3277  // The healthcheck will run again.
3278  expectClientCreate();
3279  EXPECT_CALL(*connection_, write(_, _));
3280  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3281  interval_timer_->invokeCallback();
3282
3283  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3284
3285  // Expected flow when a healthcheck times out.
3286  EXPECT_CALL(*timeout_timer_, disableTimer());
3287  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3288  connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
3289  // The healthcheck is not yet at the unhealthy threshold.
3290  EXPECT_FALSE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
3291      Host::HealthFlag::FAILED_ACTIVE_HC));
3292  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3293
3294  // The healthcheck metric results after first timeout block.
3295  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3296  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3297
3298  // The healthcheck will run again, it should be failing after this attempt.
3299  expectClientCreate();
3300  EXPECT_CALL(*connection_, write(_, _));
3301  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3302  interval_timer_->invokeCallback();
3303
3304  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3305
3306  // Expected flow when a healthcheck times out.
3307  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
3308  EXPECT_CALL(*timeout_timer_, disableTimer());
3309  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3310  connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
3311  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
3312      Host::HealthFlag::FAILED_ACTIVE_HC));
3313  EXPECT_EQ(Host::Health::Unhealthy,
3314            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3315
3316  // The healthcheck metric results after the second timeout block.
3317  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3318  EXPECT_EQ(2UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3319}
3320
3321TEST_F(TcpHealthCheckerImplTest, NoData) {
3322  InSequence s;
3323
3324  setupNoData();
3325  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3326      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3327  expectSessionCreate();
3328  expectClientCreate();
3329  EXPECT_CALL(*connection_, write(_, _)).Times(0);
3330  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3331  health_checker_->start();
3332
3333  EXPECT_CALL(*connection_, close(_));
3334  EXPECT_CALL(*timeout_timer_, disableTimer());
3335  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3336  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3337
3338  expectClientCreate();
3339  EXPECT_CALL(*connection_, write(_, _)).Times(0);
3340  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3341  interval_timer_->invokeCallback();
3342}
3343
3344TEST_F(TcpHealthCheckerImplTest, PassiveFailure) {
3345  InSequence s;
3346
3347  setupNoData();
3348  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3349      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3350  expectSessionCreate();
3351  expectClientCreate();
3352  EXPECT_CALL(*connection_, write(_, _)).Times(0);
3353  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3354  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
3355  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
3356  health_checker_->start();
3357
3358  // Do multiple passive failures. This will not reset the active HC timers.
3359  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthChecker().setUnhealthy();
3360  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthChecker().setUnhealthy();
3361  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
3362      Host::HealthFlag::FAILED_ACTIVE_HC));
3363  EXPECT_EQ(Host::Health::Unhealthy,
3364            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3365
3366  // A single success should not bring us back to healthy.
3367  EXPECT_CALL(*connection_, close(_));
3368  EXPECT_CALL(*timeout_timer_, disableTimer());
3369  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3370  connection_->raiseEvent(Network::ConnectionEvent::Connected);
3371  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
3372      Host::HealthFlag::FAILED_ACTIVE_HC));
3373  EXPECT_EQ(Host::Health::Unhealthy,
3374            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3375
3376  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.attempt").value());
3377  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3378  EXPECT_EQ(2UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3379  EXPECT_EQ(2UL, cluster_->info_->stats_store_.counter("health_check.passive_failure").value());
3380}
3381
3382TEST_F(TcpHealthCheckerImplTest, PassiveFailureCrossThreadRemoveHostRace) {
3383  InSequence s;
3384
3385  setupNoData();
3386  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3387      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3388  expectSessionCreate();
3389  expectClientCreate();
3390  EXPECT_CALL(*connection_, write(_, _)).Times(0);
3391  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3392  health_checker_->start();
3393
3394  // Do a passive failure. This will not reset the active HC timers.
3395  Event::PostCb post_cb;
3396  EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb));
3397  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthChecker().setUnhealthy();
3398
3399  // Remove before the cross thread event comes in.
3400  EXPECT_CALL(*connection_, close(_));
3401  HostVector old_hosts = std::move(cluster_->prioritySet().getMockHostSet(0)->hosts_);
3402  cluster_->prioritySet().getMockHostSet(0)->runCallbacks({}, old_hosts);
3403  post_cb();
3404
3405  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.attempt").value());
3406  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3407  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3408  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.passive_failure").value());
3409}
3410
3411TEST_F(TcpHealthCheckerImplTest, PassiveFailureCrossThreadRemoveClusterRace) {
3412  InSequence s;
3413
3414  setupNoData();
3415  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3416      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3417  expectSessionCreate();
3418  expectClientCreate();
3419  EXPECT_CALL(*connection_, write(_, _)).Times(0);
3420  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3421  health_checker_->start();
3422
3423  // Do a passive failure. This will not reset the active HC timers.
3424  Event::PostCb post_cb;
3425  EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb));
3426  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthChecker().setUnhealthy();
3427
3428  // Remove before the cross thread event comes in.
3429  EXPECT_CALL(*connection_, close(_));
3430  health_checker_.reset();
3431  post_cb();
3432
3433  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.attempt").value());
3434  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3435  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3436  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.passive_failure").value());
3437}
3438
3439TEST_F(TcpHealthCheckerImplTest, ConnectionLocalFailure) {
3440  InSequence s;
3441
3442  setupData();
3443  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3444      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3445  expectSessionCreate();
3446  expectClientCreate();
3447  EXPECT_CALL(*connection_, write(_, _));
3448  EXPECT_CALL(*timeout_timer_, enableTimer(_, _));
3449  health_checker_->start();
3450
3451  // Expect the LocalClose to be handled as a health check failure
3452  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
3453  EXPECT_CALL(*timeout_timer_, disableTimer());
3454  EXPECT_CALL(*interval_timer_, enableTimer(_, _));
3455
3456  // Raise a LocalClose that is not triggered by the health monitor itself.
3457  // e.g. a failure to setsockopt().
3458  connection_->raiseEvent(Network::ConnectionEvent::LocalClose);
3459
3460  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.attempt").value());
3461  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.success").value());
3462  EXPECT_EQ(1UL, cluster_->info_->stats_store_.counter("health_check.failure").value());
3463  EXPECT_EQ(0UL, cluster_->info_->stats_store_.counter("health_check.passive_failure").value());
3464}
3465
3466class TestGrpcHealthCheckerImpl : public GrpcHealthCheckerImpl {
3467public:
3468  using GrpcHealthCheckerImpl::GrpcHealthCheckerImpl;
3469
3470  Http::CodecClientPtr createCodecClient(Upstream::Host::CreateConnectionData& conn_data) override {
3471    auto codec_client = createCodecClient_(conn_data);
3472    return Http::CodecClientPtr(codec_client);
3473  };
3474
3475  // GrpcHealthCheckerImpl
3476  MOCK_METHOD(Http::CodecClient*, createCodecClient_, (Upstream::Host::CreateConnectionData&));
3477};
3478
3479class GrpcHealthCheckerImplTestBase : public HealthCheckerTestBase {
3480public:
3481  struct TestSession {
3482    TestSession() = default;
3483
3484    Event::MockTimer* interval_timer_{};
3485    Event::MockTimer* timeout_timer_{};
3486    Http::MockClientConnection* codec_{};
3487    Stats::IsolatedStoreImpl stats_store_;
3488    Network::MockClientConnection* client_connection_{};
3489    NiceMock<Http::MockRequestEncoder> request_encoder_;
3490    Http::ResponseDecoder* stream_response_callbacks_{};
3491    CodecClientForTest* codec_client_{};
3492  };
3493
3494  using TestSessionPtr = std::unique_ptr<TestSession>;
3495
3496  struct ResponseSpec {
3497    struct ChunkSpec {
3498      bool valid;
3499      std::vector<uint8_t> data;
3500    };
3501    static ChunkSpec invalidChunk() {
3502      ChunkSpec spec;
3503      spec.valid = false;
3504      return spec;
3505    }
3506    static ChunkSpec invalidPayload(uint8_t flags, bool valid_message) {
3507      ChunkSpec spec;
3508      spec.valid = true;
3509      spec.data = serializeResponse(grpc::health::v1::HealthCheckResponse::SERVING);
3510      spec.data[0] = flags;
3511      if (!valid_message) {
3512        const size_t kGrpcHeaderSize = 5;
3513        for (size_t i = kGrpcHeaderSize; i < spec.data.size(); i++) {
3514          // Fill payload with some random data.
3515          spec.data[i] = i % 256;
3516        }
3517      }
3518      return spec;
3519    }
3520    static ChunkSpec validChunk(grpc::health::v1::HealthCheckResponse::ServingStatus status) {
3521      ChunkSpec spec;
3522      spec.valid = true;
3523      spec.data = serializeResponse(status);
3524      return spec;
3525    }
3526
3527    static ChunkSpec servingResponse() {
3528      return validChunk(grpc::health::v1::HealthCheckResponse::SERVING);
3529    }
3530
3531    static ChunkSpec notServingResponse() {
3532      return validChunk(grpc::health::v1::HealthCheckResponse::NOT_SERVING);
3533    }
3534
3535    static std::vector<uint8_t>
3536    serializeResponse(grpc::health::v1::HealthCheckResponse::ServingStatus status) {
3537      grpc::health::v1::HealthCheckResponse response;
3538      response.set_status(status);
3539      const auto data = Grpc::Common::serializeToGrpcFrame(response);
3540      auto ret = std::vector<uint8_t>(data->length(), 0);
3541      data->copyOut(0, data->length(), &ret[0]);
3542      return ret;
3543    }
3544
3545    std::vector<std::pair<std::string, std::string>> response_headers;
3546    std::vector<ChunkSpec> body_chunks;
3547    std::vector<std::pair<std::string, std::string>> trailers;
3548  };
3549
3550  GrpcHealthCheckerImplTestBase() {
3551    EXPECT_CALL(*cluster_->info_, features())
3552        .WillRepeatedly(Return(Upstream::ClusterInfo::Features::HTTP2));
3553  }
3554
3555  void allocHealthChecker(const envoy::config::core::v3::HealthCheck& config) {
3556    health_checker_ = std::make_shared<TestGrpcHealthCheckerImpl>(
3557        *cluster_, config, dispatcher_, runtime_, random_,
3558        HealthCheckEventLoggerPtr(event_logger_));
3559  }
3560
3561  void addCompletionCallback() {
3562    health_checker_->addHostCheckCompleteCb(
3563        [this](HostSharedPtr host, HealthTransition changed_state) -> void {
3564          onHostStatus(host, changed_state);
3565        });
3566  }
3567
3568  void setupHC() {
3569    const auto config = createGrpcHealthCheckConfig();
3570    allocHealthChecker(config);
3571    addCompletionCallback();
3572  }
3573
3574  void setupHCWithUnhealthyThreshold(int value) {
3575    auto config = createGrpcHealthCheckConfig();
3576    config.mutable_unhealthy_threshold()->set_value(value);
3577    allocHealthChecker(config);
3578    addCompletionCallback();
3579  }
3580
3581  void setupServiceNameHC(const absl::optional<std::string>& authority) {
3582    auto config = createGrpcHealthCheckConfig();
3583    config.mutable_grpc_health_check()->set_service_name("service");
3584    if (authority.has_value()) {
3585      config.mutable_grpc_health_check()->set_authority(authority.value());
3586    }
3587    allocHealthChecker(config);
3588    addCompletionCallback();
3589  }
3590
3591  void setupNoReuseConnectionHC() {
3592    auto config = createGrpcHealthCheckConfig();
3593    config.mutable_reuse_connection()->set_value(false);
3594    allocHealthChecker(config);
3595    addCompletionCallback();
3596  }
3597
3598  void setupHealthCheckIntervalOverridesHC() {
3599    auto config = createGrpcHealthCheckConfig();
3600    config.mutable_interval()->set_seconds(1);
3601    config.mutable_unhealthy_interval()->set_seconds(2);
3602    config.mutable_unhealthy_edge_interval()->set_seconds(3);
3603    config.mutable_healthy_edge_interval()->set_seconds(4);
3604    config.mutable_no_traffic_interval()->set_seconds(5);
3605    config.mutable_interval_jitter()->set_seconds(0);
3606    config.mutable_unhealthy_threshold()->set_value(3);
3607    config.mutable_healthy_threshold()->set_value(3);
3608    allocHealthChecker(config);
3609    addCompletionCallback();
3610  }
3611
3612  void expectSessionCreate() {
3613    // Expectations are in LIFO order.
3614    TestSessionPtr new_test_session(new TestSession());
3615    test_sessions_.emplace_back(std::move(new_test_session));
3616    TestSession& test_session = *test_sessions_.back();
3617    test_session.timeout_timer_ = new Event::MockTimer(&dispatcher_);
3618    test_session.interval_timer_ = new Event::MockTimer(&dispatcher_);
3619    expectClientCreate(test_sessions_.size() - 1);
3620  }
3621
3622  void expectClientCreate(size_t index) {
3623    TestSession& test_session = *test_sessions_[index];
3624    test_session.codec_ = new NiceMock<Http::MockClientConnection>();
3625    test_session.client_connection_ = new NiceMock<Network::MockClientConnection>();
3626    connection_index_.push_back(index);
3627    codec_index_.push_back(index);
3628
3629    EXPECT_CALL(dispatcher_, createClientConnection_(_, _, _, _))
3630        .Times(testing::AnyNumber())
3631        .WillRepeatedly(InvokeWithoutArgs([&]() -> Network::ClientConnection* {
3632          uint32_t index = connection_index_.front();
3633          connection_index_.pop_front();
3634          return test_sessions_[index]->client_connection_;
3635        }));
3636
3637    EXPECT_CALL(*health_checker_, createCodecClient_(_))
3638        .WillRepeatedly(
3639            Invoke([&](Upstream::Host::CreateConnectionData& conn_data) -> Http::CodecClient* {
3640              uint32_t index = codec_index_.front();
3641              codec_index_.pop_front();
3642              TestSession& test_session = *test_sessions_[index];
3643              std::shared_ptr<Upstream::MockClusterInfo> cluster{
3644                  new NiceMock<Upstream::MockClusterInfo>()};
3645              Event::MockDispatcher dispatcher_;
3646
3647              test_session.codec_client_ = new CodecClientForTest(
3648                  Http::CodecClient::Type::HTTP1, std::move(conn_data.connection_),
3649                  test_session.codec_, nullptr,
3650                  Upstream::makeTestHost(cluster, "tcp://127.0.0.1:9000"), dispatcher_);
3651              return test_session.codec_client_;
3652            }));
3653  }
3654
3655  void expectStreamCreate(size_t index) {
3656    test_sessions_[index]->request_encoder_.stream_.callbacks_.clear();
3657    EXPECT_CALL(*test_sessions_[index]->codec_, newStream(_))
3658        .WillOnce(DoAll(SaveArgAddress(&test_sessions_[index]->stream_response_callbacks_),
3659                        ReturnRef(test_sessions_[index]->request_encoder_)));
3660  }
3661
3662  // Starts healthchecker and sets up timer expectations, leaving up future specification of
3663  // healthcheck response for the caller. Useful when there is only one healthcheck attempt
3664  // performed during test case (but possibly on many hosts).
3665  void expectHealthchecks(HealthTransition host_changed_state, size_t num_healthchecks) {
3666    for (size_t i = 0; i < num_healthchecks; i++) {
3667      cluster_->info_->stats().upstream_cx_total_.inc();
3668      expectSessionCreate();
3669      expectHealthcheckStart(i);
3670    }
3671    health_checker_->start();
3672
3673    EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _))
3674        .Times(num_healthchecks);
3675    EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
3676        .Times(num_healthchecks)
3677        .WillRepeatedly(Return(45000));
3678    for (size_t i = 0; i < num_healthchecks; i++) {
3679      expectHealthcheckStop(i, 45000);
3680    }
3681    EXPECT_CALL(*this, onHostStatus(_, host_changed_state)).Times(num_healthchecks);
3682  }
3683
3684  void expectSingleHealthcheck(HealthTransition host_changed_state) {
3685    cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3686        makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3687    expectHealthchecks(host_changed_state, 1);
3688  }
3689
3690  // Hides timer/stream-related boilerplate of healthcheck start.
3691  void expectHealthcheckStart(size_t index) {
3692    expectStreamCreate(index);
3693    EXPECT_CALL(*test_sessions_[index]->timeout_timer_, enableTimer(_, _));
3694  }
3695
3696  // Hides timer-related boilerplate of healthcheck stop.
3697  void expectHealthcheckStop(size_t index, int interval_ms = 0) {
3698    if (interval_ms > 0) {
3699      EXPECT_CALL(*test_sessions_[index]->interval_timer_,
3700                  enableTimer(std::chrono::milliseconds(interval_ms), _));
3701    } else {
3702      EXPECT_CALL(*test_sessions_[index]->interval_timer_, enableTimer(_, _));
3703    }
3704    EXPECT_CALL(*test_sessions_[index]->timeout_timer_, disableTimer());
3705  }
3706
3707  // Hides host status checking boilerplate when only single host is used in test.
3708  void expectHostHealthy(bool healthy) {
3709    const auto host = cluster_->prioritySet().getMockHostSet(0)->hosts_[0];
3710    if (!healthy) {
3711      EXPECT_TRUE(host->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC));
3712      EXPECT_EQ(Host::Health::Unhealthy, host->health());
3713    } else {
3714      EXPECT_EQ(Host::Health::Healthy, host->health());
3715    }
3716  }
3717
3718  void respondServiceStatus(size_t index,
3719                            grpc::health::v1::HealthCheckResponse::ServingStatus status) {
3720    respondResponseSpec(index,
3721                        ResponseSpec{{{":status", "200"}, {"content-type", "application/grpc"}},
3722                                     {ResponseSpec::validChunk(status)},
3723                                     {{"grpc-status", "0"}}});
3724  }
3725
3726  void respondResponseSpec(size_t index, ResponseSpec&& spec) {
3727    const bool trailers_empty = spec.trailers.empty();
3728    const bool end_stream_on_headers = spec.body_chunks.empty() && trailers_empty;
3729    auto response_headers = std::make_unique<Http::TestResponseHeaderMapImpl>();
3730    for (const auto& header : spec.response_headers) {
3731      response_headers->addCopy(header.first, header.second);
3732    }
3733    test_sessions_[index]->stream_response_callbacks_->decodeHeaders(std::move(response_headers),
3734                                                                     end_stream_on_headers);
3735    for (size_t i = 0; i < spec.body_chunks.size(); i++) {
3736      const bool end_stream = i == spec.body_chunks.size() - 1 && trailers_empty;
3737      const auto& chunk = spec.body_chunks[i];
3738      if (chunk.valid) {
3739        const auto data = std::make_unique<Buffer::OwnedImpl>(chunk.data.data(), chunk.data.size());
3740        test_sessions_[index]->stream_response_callbacks_->decodeData(*data, end_stream);
3741      } else {
3742        Buffer::OwnedImpl incorrect_data("incorrect");
3743        test_sessions_[index]->stream_response_callbacks_->decodeData(incorrect_data, end_stream);
3744      }
3745    }
3746    if (!trailers_empty) {
3747      auto trailers = std::make_unique<Http::TestResponseTrailerMapImpl>();
3748      for (const auto& header : spec.trailers) {
3749        trailers->addCopy(header.first, header.second);
3750      }
3751      test_sessions_[index]->stream_response_callbacks_->decodeTrailers(std::move(trailers));
3752    }
3753  }
3754
3755  void testSingleHostSuccess(const absl::optional<std::string>& authority) {
3756    std::string expected_host = cluster_->info_->name();
3757    if (authority.has_value()) {
3758      expected_host = authority.value();
3759    }
3760
3761    setupServiceNameHC(authority);
3762
3763    cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3764        makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3765    runHealthCheck(expected_host);
3766  }
3767
3768  void runHealthCheck(std::string expected_host) {
3769
3770    cluster_->info_->stats().upstream_cx_total_.inc();
3771
3772    expectSessionCreate();
3773    expectHealthcheckStart(0);
3774
3775    EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, false))
3776        .WillOnce(Invoke([&](const Http::RequestHeaderMap& headers, bool) {
3777          EXPECT_EQ(Http::Headers::get().ContentTypeValues.Grpc,
3778                    headers.ContentType()->value().getStringView());
3779          EXPECT_EQ(std::string("/grpc.health.v1.Health/Check"),
3780                    headers.Path()->value().getStringView());
3781          EXPECT_EQ(Http::Headers::get().SchemeValues.Http,
3782                    headers.Scheme()->value().getStringView());
3783          EXPECT_NE(nullptr, headers.Method());
3784          EXPECT_EQ(expected_host, headers.Host()->value().getStringView());
3785          EXPECT_EQ(std::chrono::milliseconds(1000).count(),
3786                    Envoy::Grpc::Common::getGrpcTimeout(headers).count());
3787        }));
3788    EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeData(_, true))
3789        .WillOnce(Invoke([&](Buffer::Instance& data, bool) {
3790          std::vector<Grpc::Frame> decoded_frames;
3791          Grpc::Decoder decoder;
3792          ASSERT_TRUE(decoder.decode(data, decoded_frames));
3793          ASSERT_EQ(1U, decoded_frames.size());
3794          auto& frame = decoded_frames[0];
3795          Buffer::ZeroCopyInputStreamImpl stream(std::move(frame.data_));
3796          grpc::health::v1::HealthCheckRequest request;
3797          ASSERT_TRUE(request.ParseFromZeroCopyStream(&stream));
3798          EXPECT_EQ("service", request.service());
3799        }));
3800    health_checker_->start();
3801
3802    EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _));
3803    EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _))
3804        .WillOnce(Return(45000));
3805    expectHealthcheckStop(0, 45000);
3806
3807    // Host state should not be changed (remains healthy).
3808    EXPECT_CALL(*this, onHostStatus(cluster_->prioritySet().getMockHostSet(0)->hosts_[0],
3809                                    HealthTransition::Unchanged));
3810    respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
3811    expectHostHealthy(true);
3812  }
3813
3814  MOCK_METHOD(void, onHostStatus, (HostSharedPtr host, HealthTransition changed_state));
3815
3816  std::vector<TestSessionPtr> test_sessions_;
3817  std::shared_ptr<TestGrpcHealthCheckerImpl> health_checker_;
3818  std::list<uint32_t> connection_index_{};
3819  std::list<uint32_t> codec_index_{};
3820};
3821
3822class GrpcHealthCheckerImplTest : public testing::Test, public GrpcHealthCheckerImplTestBase {};
3823
3824// Test single host check success.
3825TEST_F(GrpcHealthCheckerImplTest, Success) { testSingleHostSuccess(absl::nullopt); }
3826
3827TEST_F(GrpcHealthCheckerImplTest, SuccessWithHostname) {
3828  std::string expected_host = "www.envoyproxy.io";
3829
3830  setupServiceNameHC(absl::nullopt);
3831
3832  envoy::config::endpoint::v3::Endpoint::HealthCheckConfig health_check_config;
3833  health_check_config.set_hostname(expected_host);
3834  auto test_host = std::make_shared<HostImpl>(
3835      cluster_->info_, "", Network::Utility::resolveUrl("tcp://127.0.0.1:80"), nullptr, 1,
3836      envoy::config::core::v3::Locality(), health_check_config, 0,
3837      envoy::config::core::v3::UNKNOWN);
3838  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {test_host};
3839  runHealthCheck(expected_host);
3840}
3841
3842TEST_F(GrpcHealthCheckerImplTest, SuccessWithHostnameOverridesConfig) {
3843  std::string expected_host = "www.envoyproxy.io";
3844
3845  setupServiceNameHC("foo.com");
3846
3847  envoy::config::endpoint::v3::Endpoint::HealthCheckConfig health_check_config;
3848  health_check_config.set_hostname(expected_host);
3849  auto test_host = std::make_shared<HostImpl>(
3850      cluster_->info_, "", Network::Utility::resolveUrl("tcp://127.0.0.1:80"), nullptr, 1,
3851      envoy::config::core::v3::Locality(), health_check_config, 0,
3852      envoy::config::core::v3::UNKNOWN);
3853  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {test_host};
3854  runHealthCheck(expected_host);
3855}
3856
3857// Test single host check success with custom authority.
3858TEST_F(GrpcHealthCheckerImplTest, SuccessWithCustomAuthority) {
3859  const std::string authority = "www.envoyproxy.io";
3860  testSingleHostSuccess(authority);
3861}
3862
3863// Test host check success when gRPC response payload is split between several incoming data chunks.
3864TEST_F(GrpcHealthCheckerImplTest, SuccessResponseSplitBetweenChunks) {
3865  setupServiceNameHC(absl::nullopt);
3866  expectSingleHealthcheck(HealthTransition::Unchanged);
3867
3868  auto response_headers = std::make_unique<Http::TestResponseHeaderMapImpl>(
3869      std::initializer_list<std::pair<std::string, std::string>>{
3870          {":status", "200"},
3871          {"content-type", "application/grpc"},
3872      });
3873  test_sessions_[0]->stream_response_callbacks_->decodeHeaders(std::move(response_headers), false);
3874
3875  grpc::health::v1::HealthCheckResponse response;
3876  response.set_status(grpc::health::v1::HealthCheckResponse::SERVING);
3877  auto data = Grpc::Common::serializeToGrpcFrame(response);
3878
3879  const char* raw_data = static_cast<char*>(data->linearize(data->length()));
3880  const uint64_t chunk_size = data->length() / 5;
3881  for (uint64_t offset = 0; offset < data->length(); offset += chunk_size) {
3882    const uint64_t effective_size = std::min(chunk_size, data->length() - offset);
3883    const auto chunk = std::make_unique<Buffer::OwnedImpl>(raw_data + offset, effective_size);
3884    test_sessions_[0]->stream_response_callbacks_->decodeData(*chunk, false);
3885  }
3886
3887  auto trailers = std::make_unique<Http::TestResponseTrailerMapImpl>(
3888      std::initializer_list<std::pair<std::string, std::string>>{{"grpc-status", "0"}});
3889  test_sessions_[0]->stream_response_callbacks_->decodeTrailers(std::move(trailers));
3890
3891  expectHostHealthy(true);
3892}
3893
3894// Test host check success with multiple hosts.
3895TEST_F(GrpcHealthCheckerImplTest, SuccessWithMultipleHosts) {
3896  setupHC();
3897
3898  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3899      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80"),
3900      makeTestHost(cluster_->info_, "tcp://127.0.0.1:81")};
3901
3902  expectHealthchecks(HealthTransition::Unchanged, 2);
3903
3904  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
3905  respondServiceStatus(1, grpc::health::v1::HealthCheckResponse::SERVING);
3906  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3907  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[1]->health());
3908}
3909
3910// Test host check success with multiple hosts across multiple priorities.
3911TEST_F(GrpcHealthCheckerImplTest, SuccessWithMultipleHostSets) {
3912  setupHC();
3913
3914  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3915      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3916  cluster_->prioritySet().getMockHostSet(1)->hosts_ = {
3917      makeTestHost(cluster_->info_, "tcp://127.0.0.1:81")};
3918
3919  expectHealthchecks(HealthTransition::Unchanged, 2);
3920
3921  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
3922  respondServiceStatus(1, grpc::health::v1::HealthCheckResponse::SERVING);
3923  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
3924  EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(1)->hosts_[0]->health());
3925}
3926
3927// Test stream-level watermarks does not interfere with health check.
3928TEST_F(GrpcHealthCheckerImplTest, StreamReachesWatermarkDuringCheck) {
3929  setupHC();
3930  expectSingleHealthcheck(HealthTransition::Unchanged);
3931
3932  test_sessions_[0]->request_encoder_.stream_.runHighWatermarkCallbacks();
3933  test_sessions_[0]->request_encoder_.stream_.runLowWatermarkCallbacks();
3934
3935  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
3936  expectHostHealthy(true);
3937}
3938
3939// Test connection-level watermarks does not interfere with health check.
3940TEST_F(GrpcHealthCheckerImplTest, ConnectionReachesWatermarkDuringCheck) {
3941  setupHC();
3942  expectSingleHealthcheck(HealthTransition::Unchanged);
3943
3944  test_sessions_[0]->client_connection_->runHighWatermarkCallbacks();
3945  test_sessions_[0]->client_connection_->runLowWatermarkCallbacks();
3946
3947  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
3948  expectHostHealthy(true);
3949}
3950
3951// Test health check on host without traffic sets larger unconfigurable interval for the next check.
3952TEST_F(GrpcHealthCheckerImplTest, SuccessNoTraffic) {
3953  setupHC();
3954  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3955      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3956
3957  expectSessionCreate();
3958  expectHealthcheckStart(0);
3959  health_checker_->start();
3960
3961  // Default healthcheck interval for hosts without traffic is 60 seconds.
3962  expectHealthcheckStop(0, 60000);
3963  // Host state should not be changed (remains healthy).
3964  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
3965  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
3966  expectHostHealthy(true);
3967}
3968
3969// Test first successful check immediately makes failed host available (without 2nd probe).
3970TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedSuccessFirst) {
3971  setupHC();
3972  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3973      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
3974  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet(
3975      Host::HealthFlag::FAILED_ACTIVE_HC);
3976  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet(
3977      Host::HealthFlag::PENDING_ACTIVE_HC);
3978
3979  expectSessionCreate();
3980  expectHealthcheckStart(0);
3981  health_checker_->start();
3982
3983  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).WillOnce(Return(500));
3984  EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _));
3985  expectHealthcheckStop(0, 500);
3986  // Fast success immediately moves us to healthy.
3987  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
3988  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, true));
3989  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
3990  expectHostHealthy(true);
3991  EXPECT_FALSE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
3992      Host::HealthFlag::PENDING_ACTIVE_HC));
3993}
3994
3995// Test host recovery after first failed check requires several successful checks.
3996TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedFailFirst) {
3997  setupHC();
3998  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
3999      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4000  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet(
4001      Host::HealthFlag::FAILED_ACTIVE_HC);
4002  cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet(
4003      Host::HealthFlag::PENDING_ACTIVE_HC);
4004
4005  expectSessionCreate();
4006  expectHealthcheckStart(0);
4007  health_checker_->start();
4008
4009  // Failing first disables fast success.
4010  expectHealthcheckStop(0);
4011  // Host was unhealthy from the start, but we expect a state change due to the pending active hc
4012  // flag changing.
4013  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4014  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4015  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING);
4016  expectHostHealthy(false);
4017  EXPECT_FALSE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
4018      Host::HealthFlag::PENDING_ACTIVE_HC));
4019
4020  // Next successful healthcheck does not move host int healthy state (because we configured
4021  // healthchecker this way).
4022  expectHealthcheckStart(0);
4023  test_sessions_[0]->interval_timer_->invokeCallback();
4024
4025  expectHealthcheckStop(0);
4026  // Host still unhealthy, need yet another healthcheck.
4027  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4028  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4029  expectHostHealthy(false);
4030
4031  // 2nd successful healthcheck renders host healthy.
4032  expectHealthcheckStart(0);
4033  test_sessions_[0]->interval_timer_->invokeCallback();
4034
4035  expectHealthcheckStop(0);
4036  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4037  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
4038  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4039  expectHostHealthy(true);
4040}
4041
4042// Test host recovery after explicit check failure requires several successful checks.
4043TEST_F(GrpcHealthCheckerImplTest, GrpcHealthFail) {
4044  setupHC();
4045  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4046      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4047
4048  expectSessionCreate();
4049  expectHealthcheckStart(0);
4050  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4051  health_checker_->start();
4052
4053  // Explicit healthcheck failure immediately renders host unhealthy.
4054  expectHealthcheckStop(0);
4055  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4056  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4057  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING);
4058  expectHostHealthy(false);
4059
4060  // Next, we need 2 successful checks for host to become available again.
4061  expectHealthcheckStart(0);
4062  test_sessions_[0]->interval_timer_->invokeCallback();
4063
4064  expectHealthcheckStop(0);
4065  // Host still considered unhealthy.
4066  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4067  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4068  expectHostHealthy(false);
4069
4070  expectHealthcheckStart(0);
4071  test_sessions_[0]->interval_timer_->invokeCallback();
4072
4073  expectHealthcheckStop(0);
4074  // Host should has become healthy.
4075  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4076  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
4077  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4078  expectHostHealthy(true);
4079}
4080
4081// Test disconnects produce network-type failures which does not lead to immediate unhealthy state.
4082TEST_F(GrpcHealthCheckerImplTest, Disconnect) {
4083  setupHC();
4084  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4085      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4086
4087  expectSessionCreate();
4088  expectHealthcheckStart(0);
4089  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4090  health_checker_->start();
4091
4092  expectHealthcheckStop(0);
4093  // Network-type healthcheck failure should make host unhealthy only after 2nd event in a row.
4094  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4095  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
4096  expectHostHealthy(true);
4097
4098  expectClientCreate(0);
4099  expectHealthcheckStart(0);
4100  test_sessions_[0]->interval_timer_->invokeCallback();
4101
4102  expectHealthcheckStop(0);
4103  // Now, host should be unhealthy.
4104  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4105  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4106  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
4107  expectHostHealthy(false);
4108}
4109
4110TEST_F(GrpcHealthCheckerImplTest, Timeout) {
4111  setupHCWithUnhealthyThreshold(1);
4112  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4113      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4114  expectSessionCreate();
4115
4116  expectHealthcheckStart(0);
4117  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4118  health_checker_->start();
4119
4120  expectHealthcheckStop(0);
4121  // Unhealthy threshold is 1 so first timeout causes unhealthy
4122  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4123  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4124  test_sessions_[0]->timeout_timer_->invokeCallback();
4125  expectHostHealthy(false);
4126}
4127
4128// Test timeouts produce network-type failures which does not lead to immediate unhealthy state.
4129TEST_F(GrpcHealthCheckerImplTest, DoubleTimeout) {
4130  setupHC();
4131  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4132      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4133  expectSessionCreate();
4134
4135  expectHealthcheckStart(0);
4136  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4137  health_checker_->start();
4138
4139  expectHealthcheckStop(0);
4140  // Timeouts are considered network failures and make host unhealthy also after 2nd event.
4141  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4142  test_sessions_[0]->timeout_timer_->invokeCallback();
4143  expectHostHealthy(true);
4144
4145  expectHealthcheckStart(0);
4146  test_sessions_[0]->interval_timer_->invokeCallback();
4147
4148  expectHealthcheckStop(0);
4149  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4150  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4151  // Close connection. Timeouts and connection closes counts together.
4152  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
4153  expectHostHealthy(false);
4154}
4155
4156// Test adding and removal of hosts starts and closes healthcheck sessions.
4157TEST_F(GrpcHealthCheckerImplTest, DynamicAddAndRemove) {
4158  setupHC();
4159  health_checker_->start();
4160
4161  expectSessionCreate();
4162  expectStreamCreate(0);
4163  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4164      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4165  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4166  cluster_->prioritySet().getMockHostSet(0)->runCallbacks(
4167      {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {});
4168
4169  HostVector removed{cluster_->prioritySet().getMockHostSet(0)->hosts_.back()};
4170  cluster_->prioritySet().getMockHostSet(0)->hosts_.clear();
4171  EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_));
4172  cluster_->prioritySet().getMockHostSet(0)->runCallbacks({}, removed);
4173}
4174
4175TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) {
4176  setupHealthCheckIntervalOverridesHC();
4177  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4178      makeTestHost(cluster_->info_, "tcp://128.0.0.1:80")};
4179  expectSessionCreate();
4180  expectStreamCreate(0);
4181  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4182  health_checker_->start();
4183
4184  // First check should respect no_traffic_interval setting.
4185  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4186  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000), _));
4187  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4188  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4189  cluster_->info_->stats().upstream_cx_total_.inc();
4190
4191  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4192  // Needed after a response is sent.
4193  expectStreamCreate(0);
4194  test_sessions_[0]->interval_timer_->invokeCallback();
4195
4196  // Follow up successful checks should respect interval setting.
4197  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4198  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
4199  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4200  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4201
4202  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4203  // Needed after a response is sent.
4204  expectStreamCreate(0);
4205  test_sessions_[0]->interval_timer_->invokeCallback();
4206
4207  // Follow up successful checks should respect interval setting.
4208  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4209  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
4210  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4211  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4212
4213  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4214  // Needed after a response is sent.
4215  expectStreamCreate(0);
4216  test_sessions_[0]->interval_timer_->invokeCallback();
4217
4218  // A logical failure is not considered a network failure, therefore the unhealthy threshold is
4219  // ignored and health state changes immediately. Since the threshold is ignored, next health
4220  // check respects "unhealthy_interval".
4221  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4222  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4223  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
4224  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4225  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING);
4226
4227  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4228  // Needed after a response is sent.
4229  expectStreamCreate(0);
4230  test_sessions_[0]->interval_timer_->invokeCallback();
4231
4232  // Subsequent failing checks should respect unhealthy_interval.
4233  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4234  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
4235  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4236  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING);
4237
4238  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4239  // Needed after a response is sent.
4240  expectStreamCreate(0);
4241  test_sessions_[0]->interval_timer_->invokeCallback();
4242
4243  // Subsequent failing checks should respect unhealthy_interval.
4244  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4245  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
4246  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4247  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING);
4248
4249  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4250  // Needed after a response is sent.
4251  expectStreamCreate(0);
4252  test_sessions_[0]->interval_timer_->invokeCallback();
4253
4254  // When transitioning to a successful state, checks should respect healthy_edge_interval. Health
4255  // state should be delayed pending healthy threshold.
4256  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4257  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
4258  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4259  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4260
4261  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4262  // Needed after a response is sent.
4263  expectStreamCreate(0);
4264  test_sessions_[0]->interval_timer_->invokeCallback();
4265
4266  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4267  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
4268  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4269  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4270
4271  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4272  // Needed after a response is sent.
4273  expectStreamCreate(0);
4274  test_sessions_[0]->interval_timer_->invokeCallback();
4275
4276  // After the healthy threshold is reached, health state should change while checks should respect
4277  // the default interval.
4278  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4279  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
4280  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
4281  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4282  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4283
4284  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4285  // Needed after a response is sent.
4286  expectStreamCreate(0);
4287  test_sessions_[0]->interval_timer_->invokeCallback();
4288
4289  // Subsequent checks shouldn't change the state.
4290  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4291  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
4292  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4293  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4294
4295  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4296  // Needed after a response is sent.
4297  expectStreamCreate(0);
4298  test_sessions_[0]->interval_timer_->invokeCallback();
4299
4300  // First failed check after a run o successful ones should respect unhealthy_edge_interval. A
4301  // timeout, being a network type failure, should respect unhealthy threshold before changing the
4302  // health state.
4303  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4304  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _));
4305  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4306  test_sessions_[0]->timeout_timer_->invokeCallback();
4307
4308  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4309  // Needed after a response is sent.
4310  expectStreamCreate(0);
4311  test_sessions_[0]->interval_timer_->invokeCallback();
4312
4313  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4314  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _));
4315  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4316  test_sessions_[0]->timeout_timer_->invokeCallback();
4317
4318  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4319  // Needed after a response is sent.
4320  expectStreamCreate(0);
4321  test_sessions_[0]->interval_timer_->invokeCallback();
4322
4323  // Subsequent failing checks should respect unhealthy_interval. As the unhealthy threshold is
4324  // reached, health state should also change.
4325  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4326  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4327  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
4328  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4329  test_sessions_[0]->timeout_timer_->invokeCallback();
4330
4331  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4332  // Needed after a response is sent.
4333  expectStreamCreate(0);
4334  test_sessions_[0]->interval_timer_->invokeCallback();
4335
4336  // Remaining failing checks shouldn't change the state.
4337  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4338  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _));
4339  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4340  test_sessions_[0]->timeout_timer_->invokeCallback();
4341
4342  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4343  // Needed after a response is sent.
4344  expectStreamCreate(0);
4345  test_sessions_[0]->interval_timer_->invokeCallback();
4346
4347  // When transitioning to a successful state, checks should respect healthy_edge_interval.
4348  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4349  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
4350  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4351  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4352
4353  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4354  // Needed after a response is sent.
4355  expectStreamCreate(0);
4356  test_sessions_[0]->interval_timer_->invokeCallback();
4357
4358  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending));
4359  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _));
4360  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4361  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4362
4363  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4364  // Needed after a response is sent.
4365  expectStreamCreate(0);
4366  test_sessions_[0]->interval_timer_->invokeCallback();
4367
4368  // After the healthy threshold is reached, health state should change while checks should respect
4369  // the default interval.
4370  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed));
4371  EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false));
4372  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
4373  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4374  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4375
4376  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _));
4377  // Needed after a response is sent.
4378  expectStreamCreate(0);
4379  test_sessions_[0]->interval_timer_->invokeCallback();
4380
4381  // Subsequent checks shouldn't change the state.
4382  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4383  EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _));
4384  EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer());
4385  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4386}
4387
4388// Test connection close between checks affects nothing.
4389TEST_F(GrpcHealthCheckerImplTest, RemoteCloseBetweenChecks) {
4390  setupHC();
4391  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4392      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4393
4394  expectSessionCreate();
4395  expectHealthcheckStart(0);
4396  health_checker_->start();
4397
4398  expectHealthcheckStop(0);
4399  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4400  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4401  expectHostHealthy(true);
4402
4403  // Connection closed between checks - nothing happens, just re-create client.
4404  test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose);
4405
4406  expectClientCreate(0);
4407  expectHealthcheckStart(0);
4408  test_sessions_[0]->interval_timer_->invokeCallback();
4409
4410  expectHealthcheckStop(0);
4411  // Test host state haven't changed.
4412  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4413  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4414  expectHostHealthy(true);
4415}
4416
4417// Test that we close connections on a healthy check when reuse_connection is false.
4418TEST_F(GrpcHealthCheckerImplTest, DontReuseConnectionBetweenChecks) {
4419  setupNoReuseConnectionHC();
4420  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4421      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4422
4423  expectSessionCreate();
4424  expectHealthcheckStart(0);
4425  health_checker_->start();
4426
4427  expectHealthcheckStop(0);
4428  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4429  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4430  expectHostHealthy(true);
4431
4432  // A new client is created because we close the connection ourselves.
4433  // See GrpcHealthCheckerImplTest.RemoteCloseBetweenChecks for how this works when the remote end
4434  // closes the connection.
4435  expectClientCreate(0);
4436  expectHealthcheckStart(0);
4437  test_sessions_[0]->interval_timer_->invokeCallback();
4438
4439  expectHealthcheckStop(0);
4440  // Test host state haven't changed.
4441  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4442  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4443  expectHostHealthy(true);
4444}
4445
4446// Test UNKNOWN health status is considered unhealthy.
4447TEST_F(GrpcHealthCheckerImplTest, GrpcFailUnknown) {
4448  setupHC();
4449  expectSingleHealthcheck(HealthTransition::Changed);
4450  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4451  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4452
4453  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::UNKNOWN);
4454  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
4455      Host::HealthFlag::FAILED_ACTIVE_HC));
4456  EXPECT_EQ(Host::Health::Unhealthy,
4457            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
4458}
4459
4460// Test SERVICE_UNKNOWN health status is considered unhealthy.
4461TEST_F(GrpcHealthCheckerImplTest, GrpcFailServiceUnknown) {
4462  setupHC();
4463  expectSingleHealthcheck(HealthTransition::Changed);
4464  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4465  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4466
4467  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVICE_UNKNOWN);
4468  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
4469      Host::HealthFlag::FAILED_ACTIVE_HC));
4470  EXPECT_EQ(Host::Health::Unhealthy,
4471            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
4472}
4473
4474// Test non existent health status enum is considered unhealthy.
4475TEST_F(GrpcHealthCheckerImplTest, GrpcFailUnknownHealthStatus) {
4476  setupHC();
4477  expectSingleHealthcheck(HealthTransition::Changed);
4478  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4479  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4480
4481  respondServiceStatus(0, static_cast<grpc::health::v1::HealthCheckResponse::ServingStatus>(999));
4482  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
4483      Host::HealthFlag::FAILED_ACTIVE_HC));
4484  EXPECT_EQ(Host::Health::Unhealthy,
4485            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
4486}
4487
4488// Test receiving GOAWAY is interpreted as connection close event.
4489TEST_F(GrpcHealthCheckerImplTest, GoAwayProbeInProgress) {
4490  // FailureType::Network will be issued, it will render host unhealthy only if unhealthy_threshold
4491  // is reached.
4492  setupHCWithUnhealthyThreshold(1);
4493  expectSingleHealthcheck(HealthTransition::Changed);
4494  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4495  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4496
4497  test_sessions_[0]->codec_client_->raiseGoAway();
4498
4499  EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet(
4500      Host::HealthFlag::FAILED_ACTIVE_HC));
4501  EXPECT_EQ(Host::Health::Unhealthy,
4502            cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health());
4503}
4504
4505// Test receiving GOAWAY between checks affects nothing.
4506TEST_F(GrpcHealthCheckerImplTest, GoAwayBetweenChecks) {
4507  setupHC();
4508  cluster_->prioritySet().getMockHostSet(0)->hosts_ = {
4509      makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")};
4510
4511  expectSessionCreate();
4512  expectHealthcheckStart(0);
4513  health_checker_->start();
4514
4515  expectHealthcheckStop(0);
4516  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4517  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4518  expectHostHealthy(true);
4519
4520  // GOAWAY between checks should go unnoticed.
4521  test_sessions_[0]->codec_client_->raiseGoAway();
4522
4523  expectClientCreate(0);
4524  expectHealthcheckStart(0);
4525  test_sessions_[0]->interval_timer_->invokeCallback();
4526
4527  expectHealthcheckStop(0);
4528  // Test host state haven't changed.
4529  EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged));
4530  respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING);
4531  expectHostHealthy(true);
4532}
4533
4534class BadResponseGrpcHealthCheckerImplTest
4535    : public testing::TestWithParam<GrpcHealthCheckerImplTest::ResponseSpec>,
4536      public GrpcHealthCheckerImplTestBase {};
4537
4538INSTANTIATE_TEST_SUITE_P(
4539    BadResponse, BadResponseGrpcHealthCheckerImplTest,
4540    testing::ValuesIn(std::vector<GrpcHealthCheckerImplTest::ResponseSpec>{
4541        // Non-200 response.
4542        {
4543            {{":status", "500"}},
4544            {},
4545            {},
4546        },
4547        // Non-200 response with gRPC status.
4548        {
4549            {{":status", "500"}, {"grpc-status", "2"}},
4550            {},
4551            {},
4552        },
4553        // Missing content-type.
4554        {
4555            {{":status", "200"}},
4556            {},
4557            {},
4558        },
4559        // End stream on response headers.
4560        {
4561            {{":status", "200"}, {"content-type", "application/grpc"}},
4562            {},
4563            {},
4564        },
4565        // Non-OK gRPC status in headers.
4566        {
4567            {{":status", "200"}, {"content-type", "application/grpc"}, {"grpc-status", "2"}},
4568            {},
4569            {},
4570        },
4571        // Non-OK gRPC status
4572        {
4573            {{":status", "200"}, {"content-type", "application/grpc"}},
4574            {GrpcHealthCheckerImplTest::ResponseSpec::servingResponse()},
4575            {{"grpc-status", "2"}},
4576        },
4577        // Missing body.
4578        {
4579            {{":status", "200"}, {"content-type", "application/grpc"}, {"grpc-status", "0"}},
4580            {},
4581            {},
4582        },
4583        // Compressed body.
4584        {
4585            {{":status", "200"}, {"content-type", "application/grpc"}},
4586            {GrpcHealthCheckerImplTest::ResponseSpec::invalidPayload(Grpc::GRPC_FH_COMPRESSED,
4587                                                                     true)},
4588            {},
4589        },
4590        // Invalid proto message.
4591        {
4592            {{":status", "200"}, {"content-type", "application/grpc"}},
4593            {GrpcHealthCheckerImplTest::ResponseSpec::invalidPayload(Grpc::GRPC_FH_DEFAULT, false)},
4594            {},
4595        },
4596        // Duplicate response.
4597        {
4598            {{":status", "200"}, {"content-type", "application/grpc"}},
4599            {GrpcHealthCheckerImplTest::ResponseSpec::servingResponse(),
4600             GrpcHealthCheckerImplTest::ResponseSpec::servingResponse()},
4601            {},
4602        },
4603        // Invalid response.
4604        {
4605            {{":status", "200"}, {"content-type", "application/grpc"}},
4606            {GrpcHealthCheckerImplTest::ResponseSpec::invalidChunk()},
4607            {},
4608        },
4609        // No trailers.
4610        {
4611            {{":status", "200"}, {"content-type", "application/grpc"}},
4612            {GrpcHealthCheckerImplTest::ResponseSpec::servingResponse()},
4613            {},
4614        },
4615        // No gRPC status in trailer.
4616        {
4617            {{":status", "200"}, {"content-type", "application/grpc"}},
4618            {GrpcHealthCheckerImplTest::ResponseSpec::servingResponse()},
4619            {{"some-header", "1"}},
4620        },
4621        // Invalid gRPC status.
4622        {
4623            {{":status", "200"}, {"content-type", "application/grpc"}},
4624            {GrpcHealthCheckerImplTest::ResponseSpec::servingResponse()},
4625            {{"grpc-status", "invalid"}},
4626        },
4627    }));
4628
4629// Test different cases of invalid gRPC response makes host unhealthy.
4630TEST_P(BadResponseGrpcHealthCheckerImplTest, GrpcBadResponse) {
4631  setupHC();
4632  expectSingleHealthcheck(HealthTransition::Changed);
4633  EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true));
4634  EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _));
4635
4636  ResponseSpec spec = GetParam();
4637  respondResponseSpec(0, std::move(spec));
4638  expectHostHealthy(false);
4639}
4640
4641TEST(Printer, HealthStatePrinter) {
4642  std::ostringstream healthy;
4643  healthy << HealthState::Healthy;
4644  EXPECT_EQ("Healthy", healthy.str());
4645
4646  std::ostringstream unhealthy;
4647  unhealthy << HealthState::Unhealthy;
4648  EXPECT_EQ("Unhealthy", unhealthy.str());
4649}
4650
4651TEST(Printer, HealthTransitionPrinter) {
4652  std::ostringstream changed;
4653  changed << HealthTransition::Changed;
4654  EXPECT_EQ("Changed", changed.str());
4655
4656  std::ostringstream unchanged;
4657  unchanged << HealthTransition::Unchanged;
4658  EXPECT_EQ("Unchanged", unchanged.str());
4659}
4660
4661TEST(HealthCheckEventLoggerImplTest, All) {
4662  AccessLog::MockAccessLogManager log_manager;
4663  std::shared_ptr<AccessLog::MockAccessLogFile> file(new AccessLog::MockAccessLogFile());
4664  EXPECT_CALL(log_manager, createAccessLog("foo")).WillOnce(Return(file));
4665
4666  std::shared_ptr<MockHostDescription> host(new NiceMock<MockHostDescription>());
4667  NiceMock<MockClusterInfo> cluster;
4668  ON_CALL(*host, cluster()).WillByDefault(ReturnRef(cluster));
4669
4670  Event::SimulatedTimeSystem time_system;
4671  // This is rendered as "2009-02-13T23:31:31.234Z".a
4672  time_system.setSystemTime(std::chrono::milliseconds(1234567891234));
4673
4674  HealthCheckEventLoggerImpl event_logger(log_manager, time_system, "foo");
4675
4676  EXPECT_CALL(*file, write(absl::string_view{
4677                         "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
4678                         "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
4679                         "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
4680                         "cluster\",\"eject_unhealthy_event\":{\"failure_type\":\"ACTIVE\"},"
4681                         "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
4682  event_logger.logEjectUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE);
4683
4684  EXPECT_CALL(*file, write(absl::string_view{
4685                         "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
4686                         "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
4687                         "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
4688                         "cluster\",\"add_healthy_event\":{\"first_check\":false},\"timestamp\":"
4689                         "\"2009-02-13T23:31:31.234Z\"}\n"}));
4690  event_logger.logAddHealthy(envoy::data::core::v3::HTTP, host, false);
4691
4692  EXPECT_CALL(*file, write(absl::string_view{
4693                         "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
4694                         "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
4695                         "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
4696                         "cluster\",\"health_check_failure_event\":{\"failure_type\":\"ACTIVE\","
4697                         "\"first_check\":false},"
4698                         "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
4699  event_logger.logUnhealthy(envoy::data::core::v3::HTTP, host, envoy::data::core::v3::ACTIVE,
4700                            false);
4701
4702  EXPECT_CALL(*file, write(absl::string_view{
4703                         "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
4704                         "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
4705                         "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
4706                         "cluster\",\"degraded_healthy_host\":{},"
4707                         "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
4708  event_logger.logDegraded(envoy::data::core::v3::HTTP, host);
4709
4710  EXPECT_CALL(*file, write(absl::string_view{
4711                         "{\"health_checker_type\":\"HTTP\",\"host\":{\"socket_address\":{"
4712                         "\"protocol\":\"TCP\",\"address\":\"10.0.0.1\",\"resolver_name\":\"\","
4713                         "\"ipv4_compat\":false,\"port_value\":443}},\"cluster_name\":\"fake_"
4714                         "cluster\",\"no_longer_degraded_host\":{},"
4715                         "\"timestamp\":\"2009-02-13T23:31:31.234Z\"}\n"}));
4716  event_logger.logNoLongerDegraded(envoy::data::core::v3::HTTP, host);
4717}
4718
4719// Validate that the proto constraints don't allow zero length edge durations.
4720TEST(HealthCheckProto, Validation) {
4721  {
4722    const std::string yaml = R"EOF(
4723    timeout: 1s
4724    interval: 1s
4725    healthy_threshold: 1
4726    unhealthy_threshold: 1
4727    no_traffic_interval: 0s
4728    http_health_check:
4729      service_name_matcher:
4730        prefix: locations
4731      path: /healthcheck
4732    )EOF";
4733    envoy::config::core::v3::HealthCheck health_check_proto;
4734    EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException,
4735                            "Proto constraint validation failed.*value must be greater than.*");
4736  }
4737  {
4738    const std::string yaml = R"EOF(
4739    timeout: 1s
4740    interval: 1s
4741    healthy_threshold: 1
4742    unhealthy_threshold: 1
4743    unhealthy_interval: 0s
4744    http_health_check:
4745      service_name_matcher:
4746        prefix: locations
4747      path: /healthcheck
4748    )EOF";
4749    envoy::config::core::v3::HealthCheck health_check_proto;
4750    EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException,
4751                            "Proto constraint validation failed.*value must be greater than.*");
4752  }
4753  {
4754    const std::string yaml = R"EOF(
4755    timeout: 1s
4756    interval: 1s
4757    healthy_threshold: 1
4758    unhealthy_threshold: 1
4759    unhealthy_edge_interval: 0s
4760    http_health_check:
4761      service_name_matcher:
4762        prefix: locations
4763      path: /healthcheck
4764    )EOF";
4765    envoy::config::core::v3::HealthCheck health_check_proto;
4766    EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException,
4767                            "Proto constraint validation failed.*value must be greater than.*");
4768  }
4769  {
4770    const std::string yaml = R"EOF(
4771    timeout: 1s
4772    interval: 1s
4773    healthy_threshold: 1
4774    unhealthy_threshold: 1
4775    healthy_edge_interval: 0s
4776    http_health_check:
4777      service_name_matcher:
4778        prefix: locations
4779      path: /healthcheck
4780    )EOF";
4781    envoy::config::core::v3::HealthCheck health_check_proto;
4782    EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException,
4783                            "Proto constraint validation failed.*value must be greater than.*");
4784  }
4785  {
4786    const std::string yaml = R"EOF(
4787    timeout: 1s
4788    interval: 1s
4789    unhealthy_threshold: 1
4790    http_health_check:
4791      service_name_matcher:
4792        prefix: locations
4793      path: /healthcheck
4794    )EOF";
4795    envoy::config::core::v3::HealthCheck health_check_proto;
4796    EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException,
4797                            "Proto constraint validation failed.*value is required.*");
4798  }
4799  {
4800    const std::string yaml = R"EOF(
4801    timeout: 1s
4802    interval: 1s
4803    healthy_threshold: 1
4804    http_health_check:
4805      service_name_matcher:
4806        prefix: locations
4807      path: /healthcheck
4808    )EOF";
4809    envoy::config::core::v3::HealthCheck health_check_proto;
4810    EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException,
4811                            "Proto constraint validation failed.*value is required.*");
4812  }
4813}
4814
4815} // namespace
4816} // namespace Upstream
4817} // namespace Envoy