Log streaming: New Relic OTLP

1. Review the information in our guide to setting up remote log streaming.
2. In the New Relic OTLP area, click Create endpoint.
3. Fill out the Create a New Relic OTLP endpoint fields as follows:
   • In the Name field, enter a human-readable name for the endpoint.
   • In the License key / Insert key field, enter your New Relic license key or Insert API key.
   • From the Region controls, select the region to stream logs to.
4. (Optional) If you are using New Relic Infinite Tracing, click Advanced options and enter your New Relic Trace Observer URL in the Trace Observer URL field.
5. Click Create to create the new logging endpoint.
6. Click Activate to deploy your configuration changes.
7. Follow the instructions below to instrument your VCL service.

Instrumenting your VCL service

After setting up your endpoint, you will need to add custom VCL to gather the data and generate Open Telemetry traces at the edge.

1. Place the following functions at the top of your custom VCL file.

   1
   2
   3
   4
   5
   6
   7
   8
   9
   10
   11
   12
   13
   14
   15
   16
   17
   18
   19
   20
   21
   22
   23
   24
   25
   26
   27
   28
   29
   30
   31
   32
   33
   34
   35
   36
   37
   38
   39
   40
   41
   42
   43
   44
   45
   46
   47
   48
   49
   50
   51
   52
   53
   54
   55
   56
   57
   58
   59
   60
   61
   62
   63
   64
   65

   sub time_now_ns STRING {
     declare local var.time_now INTEGER;
     set var.time_now = std.atoi(time.start.usec);
     set var.time_now += std.atoi(time.elapsed.usec);
     set var.time_now *= 1000;
     return var.time_now;
   }
   sub random_8bit_identifier STRING {
     declare local var.id STRING;
     set var.id = randomstr(16, "0123456789abcdef");
     return var.id;
   }
   sub random_16bit_identifier STRING {
     declare local var.id STRING;
     set var.id = randomstr(32, "0123456789abcdef");
     return var.id;
   }
   
   sub otel_resource STRING {
      declare local var.str STRING;
      set var.str = {"{ "attributes": [ "}
      {"{ "key": "service.name",           "value": { "stringValue": "Fastly www"    } }, "}
      {"{ "key": "telemetry.sdk.language", "value": { "stringValue": "vcl"           } }, "}
      {"{ "key": "telemetry.sdk.name",     "value": { "stringValue": "opentelemetry" } }, "}
      {"{ "key": "telemetry.sdk.version",  "value": { "stringValue": "1.0.1"         } }, "}
      {"{ "key": "host.name",              "value": { "stringValue": ""} server.identity {"" } }"}
      {"], "droppedAttributesCount": 0 }"};
      return var.str;
   }
   
   sub otel_attributes_general STRING {
      declare local var.data STRING;
      set var.data = ""
      {"{ "key": "http.method",    "value": { "stringValue": ""} req.method {"" } },"}
      {"{ "key": "http.target",    "value": { "stringValue": ""} req.url {"" } },"}
      {"{ "key": "http.host",      "value": { "stringValue": ""} req.http.host {"" } },"}
      {"{ "key": "http.protocol",  "value": { "stringValue": ""} req.protocol {"" } },"}
      {"{ "key": "http.client_ip", "value": { "stringValue": ""} client.ip {"" } },"}
   
      {"{ "key": "fastly.restarts",     "value": { "stringValue": ""} req.restarts {"" } },"}
      {"{ "key": "fastly.visits_this_service", "value": { "stringValue": ""} fastly.ff.visits_this_service {"" } },"}
      {"{ "key": "fastly.server_role",  "value": { "stringValue": ""} req.http.x-trace-server-role {"" } },"}
      {"{ "key": "fastly.server_ip",    "value": { "stringValue": ""} server.ip {"" } },"}
      {"{ "key": "fastly.server_id",    "value": { "stringValue": ""} server.identity {"" } },"}
      {"{ "key": "fastly.server_role",  "value": { "stringValue": ""} req.http.x-trace-server-role {"" } },"}
      {"{ "key": "fastly.vcl_version",  "value": { "stringValue": ""} req.vcl.version {"" } },"}
      {"{ "key": "fastly.pop",          "value": { "stringValue": ""} server.datacenter {"" } },"}
      {"{ "key": "fastly.workspace.overflowed",  "value": { "stringValue": ""} workspace.overflowed {"" } },"}
      {"{ "key": "fastly.workspace.bytes_total", "value": { "stringValue": ""} workspace.bytes_total {"" } },"}
      {"{ "key": "fastly.workspace.bytes_free",  "value": { "stringValue": ""} workspace.bytes_free {"" } },"}
      ;
      return var.data;
   }
   
   sub telem_start_backend_fetch {
     set bereq.http.traceparent = "00-" req.http.x-trace-id + "-" + req.http.x-trace-vcl-span-id "-01";
   
     # Avoid leaking internal headers to backends
     unset bereq.http.x-trace-id;
     unset bereq.http.x-trace-parent-span-id;
     unset bereq.http.x-trace-server-role;
   
     # Leapfrog cloud service infra that creates 'ghost spans'
     set bereq.http.x-traceparent = bereq.http.traceparent;
   }

2. Add the following VCL to the start of the vcl_recv subroutine to record a span for the whole of a request's lifetime within a Fastly POP.

   1
   2
   3
   4
   5
   6
   7
   8
   9
   10

   if (req.restarts == 0) {
     set req.http.x-trace-vcl-span-id = random_8bit_identifier();
     if (req.http.traceparent ~ "^\d+-(\w+)-(\w+)-\d+$") {
       set req.http.x-trace-id = re.group.1;
       set req.http.x-trace-parent-span-id = re.group.2;
     } else {
       set req.http.x-trace-id = random_16bit_identifier();
     }
     set req.http.x-trace-server-role = if (fastly.ff.visits_this_service == 0, "edge", "shield");
   }

3. Add the following VCL to the end of the vcl_log subroutine, replacing otel_collector_http with the name of your endpoint.

   1
   2
   3
   4
   5
   6
   7
   8
   9
   10
   11
   12
   13
   14
   15
   16
   17
   18
   19
   20
   21
   22
   23
   24
   25
   26
   27
   28
   29
   30
   31
   32

   declare local var.otel_resource STRING;
   declare local var.otel_attribs STRING;
   declare local var.time_start_ns STRING;
   declare local var.time_now_ns STRING;
   set var.time_start_ns = time.start.usec "000";
   set var.time_now_ns = time_now_ns();
   set var.otel_resource = otel_resource();
   set var.otel_attribs = otel_attributes_general();
   log "syslog " req.service_id " otel_collector_http :: "
     {"{ "resourceSpans": [ { "}
       {""resource": "} var.otel_resource {", "}
       {""instrumentationLibrarySpans": [ { "spans": [ { "}
         {""traceId": ""} req.http.x-trace-id {"", "}
         {""spanId": ""} req.http.x-trace-vcl-span-id {"", "}
         if(req.http.x-trace-parent-span-id,
           {""parentSpanId": ""} req.http.x-trace-parent-span-id {"", "},
         "")
         {""name": "Fastly request processing", "}
         {""kind": 1, "}
         {""startTimeUnixNano": "} var.time_start_ns {", "}
         {""endTimeUnixNano": "} var.time_now_ns {", "}
         {""attributes": [ "}
           var.otel_attribs
           {"{ "key": "http.user_agent", "value": { "stringValue": ""} req.http.User-Agent {"" } }, "}
           {"{ "key": "http.status_code", "value": { "stringValue": ""} resp.status {"" } }"}
         {"], "}
         {""status": { "code":"} if (fastly_info.state ~ "ERROR", "2", "0") {" }, "}
         {""links": [], "}
         {""droppedLinksCount": 0"}
       {"} ] } ]"}
     {"} ] }"}
   ;

4. Add a call to telem_start_backend_fetch and place it in the vcl_miss and vcl_pass subroutines to see spans that happen in backend systems as children of the Fastly spans.

5. Activate the service.