001package org.avaje.metric.elastic;
002
003import okhttp3.MediaType;
004import okhttp3.OkHttpClient;
005import okhttp3.Request;
006import okhttp3.RequestBody;
007import okhttp3.Response;
008import org.avaje.metric.report.MetricReporter;
009import org.avaje.metric.report.ReportMetrics;
010import org.slf4j.Logger;
011import org.slf4j.LoggerFactory;
012
013import java.io.File;
014import java.io.FileWriter;
015import java.io.IOException;
016import java.io.StringWriter;
017import java.net.ConnectException;
018import java.net.SocketTimeoutException;
019import java.net.UnknownHostException;
020import java.nio.file.Files;
021import java.time.LocalDate;
022import java.time.format.DateTimeFormatter;
023import java.time.format.DateTimeFormatterBuilder;
024import java.util.List;
025import java.util.concurrent.TimeUnit;
026
027/**
028 * Http(s) based Reporter that sends JSON formatted metrics directly to Elastic.
029 */
030public class ElasticHttpReporter implements MetricReporter {
031
032  private static final Logger logger = LoggerFactory.getLogger(ElasticHttpReporter.class);
033
034  private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8");
035
036  private static final DateTimeFormatter todayFormat
037    = new DateTimeFormatterBuilder()
038    .appendPattern("yyyy.MM.dd")
039    .toFormatter();
040
041
042  private final File directory;
043
044  private final OkHttpClient client;
045
046  private final String bulkUrl;
047
048  private final ElasticReporterConfig config;
049
050  public ElasticHttpReporter(ElasticReporterConfig config) {
051    this.client = getClient(config);
052    this.config = config;
053    this.bulkUrl = config.getUrl() + "/_bulk";
054    this.directory = checkDirectory(config.getDirectory());
055
056    // put the template to elastic if it is not already there
057    new TemplateApply(client, config.getUrl(), config.getTemplateName()).run();
058  }
059
060  private File checkDirectory(String directory) {
061    File dir = new File(directory);
062    if (!dir.exists() && !dir.mkdirs()) {
063      throw new IllegalStateException("Unable to access or create directory [" + directory + "]");
064    }
065    return dir;
066  }
067
068  private OkHttpClient getClient(ElasticReporterConfig config) {
069
070    OkHttpClient client = config.getClient();
071    if (client != null) {
072      return client;
073    } else {
074      return new OkHttpClient.Builder()
075        .connectTimeout(config.getConnectTimeout(), TimeUnit.SECONDS)
076        .readTimeout(config.getReadTimeout(), TimeUnit.SECONDS)
077        .writeTimeout(config.getWriteTimeout(), TimeUnit.SECONDS)
078        .build();
079    }
080  }
081
082  /**
083   * Send the non-empty metrics that were collected to the remote repository.
084   */
085  @Override
086  public void report(ReportMetrics reportMetrics) {
087
088    if (reportMetrics.getMetrics().isEmpty()) {
089      return;
090    }
091
092    StringWriter writer = new StringWriter(1000);
093    BulkJsonWriteVisitor jsonVisitor = new BulkJsonWriteVisitor(writer, reportMetrics, config, today());
094    try {
095      jsonVisitor.write();
096    } catch (IOException e) {
097      logger.error("Failed to write Bulk JSON for metrics", e);
098      return;
099    }
100    String bulkJson = writer.toString();
101    if (!bulkJson.isEmpty()) {
102      sendMetrics(bulkJson, true);
103    }
104  }
105
106  /**
107   * Send the bulk message to ElasticSearch.
108   */
109  private void sendMetrics(String bulkMessage, boolean withQueued) {
110    String json = bulkMessage;
111    if (logger.isTraceEnabled()) {
112      logger.trace("Sending:\n{}", json);
113    }
114
115    RequestBody body = RequestBody.create(JSON, json);
116    Request request = new Request.Builder()
117      .url(bulkUrl)
118      .post(body)
119      .build();
120
121    try {
122      try (Response response = client.newCall(request).execute()) {
123        if (!response.isSuccessful()) {
124          logger.warn("Unsuccessful sending metrics payload to server - {}", response.body().string());
125          storeJsonForResend(json);
126        } else {
127          if (logger.isTraceEnabled()) {
128            logger.trace("Bulk Response - {}", response.body().string());
129          }
130          if (withQueued) {
131            sendQueued();
132          }
133        }
134      }
135
136    } catch (UnknownHostException e) {
137      logger.info("UnknownHostException trying to sending metrics to server: " + e.getMessage());
138      storeJsonForResend(json);
139
140    } catch (ConnectException | SocketTimeoutException e) {
141      logger.info("Connection error sending metrics to server: " + e.getMessage());
142      storeJsonForResend(json);
143
144    } catch (Exception e) {
145      logger.warn("Unexpected error sending metrics to server, metrics queued to be sent later", e);
146      storeJsonForResend(json);
147    }
148  }
149
150  /**
151   * Send any metrics files that have been queued (as they failed initial send to elasticsearch).
152   */
153  private void sendQueued() {
154
155    File[] files = directory.listFiles(pathname -> pathname.getName().endsWith(".metric"));
156    if (files == null) {
157      return;
158    }
159    for (File heldFile : files) {
160      try {
161        sendMetrics(readQueuedFile(heldFile), false);
162        if (!heldFile.delete()) {
163          logger.error("Sent but unable to deleted queued metrics file, possible duplicate metrics for file:{}", heldFile);
164        } else {
165          logger.info("Sent queued metrics file {}", heldFile.getName());
166        }
167      } catch (IOException e) {
168        // just successfully sent metrics so not really expecting this
169        logger.warn("Failed to sent queued metrics file " + heldFile.getName(), e);
170        return;
171      }
172    }
173  }
174
175  /**
176   * Read and return the content from queued metrics file.
177   */
178  private String readQueuedFile(File heldFile) throws IOException {
179    StringBuilder sb = new StringBuilder(1000);
180    List<String> lines = Files.readAllLines(heldFile.toPath());
181    for (String line : lines) {
182      sb.append(line).append("\n");
183    }
184    return sb.toString();
185  }
186
187  private String today() {
188    return todayFormat.format(LocalDate.now());
189  }
190
191  protected void storeJsonForResend(String json) {
192    try {
193      // will be unique file name
194      File out = new File(directory, "metrics-" + System.currentTimeMillis() + ".metric");
195      FileWriter fw = new FileWriter(out);
196      fw.write(json);
197      fw.flush();
198      fw.close();
199    } catch (IOException e) {
200      logger.warn("Failed to store metrics file for resending", e);
201    }
202  }
203
204  @Override
205  public void cleanup() {
206    // Do nothing
207  }
208
209}