var Component=(()=>{var d=Object.create;var o=Object.defineProperty;var l=Object.getOwnPropertyDescriptor;var c=Object.getOwnPropertyNames;var u=Object.getPrototypeOf,m=Object.prototype.hasOwnProperty;var a=n=>o(n,"__esModule",{value:!0});var p=(n,e)=>()=>(e||n((e={exports:{}}).exports,e),e.exports),g=(n,e)=>{a(n);for(var r in e)o(n,r,{get:e[r],enumerable:!0})},f=(n,e,r)=>{if(e&&typeof e=="object"||typeof e=="function")for(let i of c(e))!m.call(n,i)&&i!=="default"&&o(n,i,{get:()=>e[i],enumerable:!(r=l(e,i))||r.enumerable});return n},w=n=>f(a(o(n!=null?d(u(n)):{},"default",n&&n.__esModule&&"default"in n?{get:()=>n.default,enumerable:!0}:{value:n,enumerable:!0})),n);var h=p((S,s)=>{s.exports=_jsx_runtime});var A={};g(A,{default:()=>b,frontmatter:()=>y});var t=w(h()),y={title:"Building Shoreline's Azure Agent During My Summer Internship",date:new Date(1632484401569),author:"Tanay Menezes",featuredImage:"/uploads/posts/shoreline-azure.png",excerpt:"Important lessons and valuable experiences while developing Shoreline's Azure Agent."};function v(n){let e=Object.assign({p:"p",h2:"h2",a:"a",h3:"h3",img:"img",em:"em",ol:"ol",li:"li"},n.components),{wrapper:r}=e,i=(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(e.p,{children:"As a freshman in college majoring in computer science, my extracurricular development experience prior to my Shoreline internship was limited to personal projects and apps I had helped build as a team in various student organizations. So coming into my summer internship at Shoreline, I was excited to have my first opportunity to work in the industry at a startup to help develop a real-world product. During my ten weeks at Shoreline, I learned a lot, gained valuable experience, and met some great people."}),`
`,(0,t.jsx)(e.h2,{id:"my-work-at-shoreline",children:"My Work at Shoreline"}),`
`,(0,t.jsxs)(e.p,{children:["For context, Shoreline provides real-time automation and control for cloud operations. The ",(0,t.jsx)(e.a,{href:"https://docs.shoreline.io/platform/agents",children:"Shoreline Agent"})," is a small process running in the background of all monitored hosts, aggregating metrics data and allowing customers to query their fleet ",(0,t.jsx)(e.a,{href:"https://docs.shoreline.io/platform/resources",children:"Resources"})," in real-time."]}),`
`,(0,t.jsx)(e.h3,{id:"performance-improvements",children:"Performance Improvements"}),`
`,(0,t.jsx)(e.p,{children:`Therefore, my first couple of assignments at Shoreline involved performance improvements in the execution time of large-scale distributed metric queries and Linux commands. The core issue with the current implementation was that the
backend took about 6-7 seconds to respond after a customer entered their query into the CLI. The diagram below shows the general path of how a metric query executes.`}),`
`,(0,t.jsx)(e.p,{children:(0,t.jsx)(e.img,{src:"/uploads/posts/tanay-azure-agent/shoreline-metric-query-execution-steps.png",alt:"shoreline-metric-query-execution-steps",width:"1999",height:"648"})}),`
`,(0,t.jsx)("div",{className:"text-center text-sm",children:(0,t.jsx)(e.em,{children:"Shoreline metric query execution steps"})}),`
`,(0,t.jsxs)(e.ol,{children:[`
`,(0,t.jsx)(e.li,{children:"The Agents receive and execute the customer's metric query (MQ)."}),`
`,(0,t.jsx)(e.li,{children:"The merged result (MSLE) is decoded and forwarded to any additional metric queries, which created significant bottlenecks as fleet size increased."}),`
`,(0,t.jsx)(e.li,{children:"The process repeats until all queries resolve and the customer sees the final output."}),`
`]}),`
`,(0,t.jsx)(e.p,{children:"The initial scope with details about all the resources were being sent to all the Agents and they were sending it back to the Backend. Pruning the scope so that it propagated only the required data for each step improved the performance significantly. After scalability testing on a 200-Agent environment, the metric queries that once took 6-7 seconds were down to 1-2 seconds."}),`
`,(0,t.jsx)(e.h3,{id:"shoreline-azure-agent",children:"Shoreline Azure Agent"}),`
`,(0,t.jsx)(e.p,{children:'Beyond performance improvements, my main project for the summer was starting the development of the Shoreline Azure Agent. Before this project, the Shoreline Agent was only available to customers who had their machines running on AWS. When I look back on my internship at Shoreline, it was a great company to work at because I did significant work, as opposed to low-level typical "intern" tasks that no one else wanted to do. In addition, as a rising startup company about to emerge from stealth mode, the magnitude of this project was pretty significant - it would open up a new set of customers because Microsoft Azure is the second most popular cloud provider.'}),`
`,(0,t.jsx)(e.p,{children:"Azure is slightly different from AWS: the equivalent of an AWS EC2 instance is an Azure VM. For example, Azure uses a resource hierarchy account, subscription, resource group, and resource, while AWS uses account, region, and resource. There are also many differences in the instance metadata tags (tags tied to a VM instance such as name, location, etc.) between Azure VMs and EC2 instances. These differences and more had to be accounted for and kept in mind when modifying the Shoreline agent to support Azure VMs."}),`
`,(0,t.jsx)(e.p,{children:`Before a customer can monitor their fleet, their hosts must be "discovered" by the Shoreline Agent running on the machine and then "registered" internally in Shoreline's database. So naturally, this was the first step of developing the Azure Agent. After discovery, the first thing I implemented was making sure the Agent was on an Azure VM instead of a different cloud provider. I did this by using the attested metadata URL to receive a signed certificate that contained VM-related details.`}),`
`,(0,t.jsx)(e.p,{children:'Then, I added the tags collection implementation by calling the "instance metadata service," which returns all of the metadata tags for the VM. Instance metadata tags worked slightly differently between AWS and Azure: in AWS, each tag is retrieved from a request to a specific URL, and tags are retrieved that way individually. In Azure, one batch call retrieves all of the tags at once in a JSON response, which can then be parsed and handled internally.'}),`
`,(0,t.jsx)(e.p,{children:"There are also differences in provider tags: for example, in AWS, region is the tag used to specify where the actual provisioned hardware is, while in Azure, the equivalent tag is called location. There were many minor differences, so I refactored the code to make it more modular. Azure tag retrieval and AWS tag retrieval implementations had dedicated modules. In contrast, a shared module exploited them to support all hosts. This implementation abstracted the differences in cloud providers in the dedicated modules while keeping common functionalities such as caching tags in the shared module. This configuration also eases development for supporting more cloud providers in the future. After implementing Azure registration, we had an Azure bug hunt (a meeting to test some new functionality and find possible bugs). The first Shoreline bug hunt involved managing a multi-cloud environment."}),`
`,(0,t.jsx)(e.p,{children:(0,t.jsx)(e.img,{src:"/uploads/posts/tanay-azure-agent/multi-provider-monitor.png",alt:"multi-provider-monitor",width:"1999",height:"645"})}),`
`,(0,t.jsx)("div",{className:"text-center text-sm",children:(0,t.jsx)(e.em,{children:"A Shoreline cluster with multi-provider monitoring"})}),`
`,(0,t.jsx)(e.p,{children:"After registration, I worked on deregistration: the process of removing a host once it is stopped or terminated. This entailed some challenges that were not present with deregistration for AWS. For example, there's no Azure SDK for Elixir, which meant that to get the hosts to deregister, we had to use the Azure SDK for Go in the scraper service. Through this process, I learned how to use a gRPC API and protocol buffers for communication across microservices. In addition to the actual implementation, we also had to maintain backward compatibility so that no older agents were attempting to execute a deprecated function."}),`
`,(0,t.jsx)(e.h2,{id:"experienceculture",children:"Experience/Culture"}),`
`,(0,t.jsx)(e.p,{children:"At other companies, training and onboarding may take up much of a new hire's time, and it may be months before they can change code on an existing product. However, at Shoreline, I contributed to the codebase in just my second week of work. Working at Shoreline was something I looked forward to every day because of the interesting work and the people I got to work and interact with."}),`
`,(0,t.jsx)(e.p,{children:"As an intern, I had one-on-one chats with various high-ranking members of the company, where I got to know a little bit more about Shoreline and ask any questions I had throughout my internship. I not only learned more about work and life at Shoreline, but I also had the chance to ask questions about advice on school and a career in the industry, as everyone I had talked to had a diverse career background. Getting to pick the brains of these Shoreliners for tips and advice through these one-on-ones was very helpful and informative."}),`
`,(0,t.jsx)(e.p,{children:"In addition, the weekly new-hire/intern lunches and group lunches were something I always looked forward to, as it was fun to chat and get to know all of the Shoreliners. As a new intern joining an already existing team, I felt welcomed and never overwhelmed as everyone made it clear they were always willing to help if I needed it."}),`
`,(0,t.jsx)(e.p,{children:"This help extended to the technical side of work as well. Every intern is assigned to a team related to their main project and is paired with a mentor, providing a reliable and robust support system of Shoreliners always willing to help if I got stuck somewhere. Coming into the internship, I had no experience related to cloud computing or distributed systems. Still, throughout the internship, I ended up learning so much with the help of my mentor and team: multiple languages, cloud providers, Kubernetes, CI/CD, microservices, and more."})]});return r?(0,t.jsx)(r,Object.assign({},n,{children:i})):i}var b=v;return A;})();
;return Component.default;

Building Shoreline's Azure Agent During My Summer Internship

var Component=(()=>{var c=Object.create;var a=Object.defineProperty;var h=Object.getOwnPropertyDescriptor;var d=Object.getOwnPropertyNames;var p=Object.getPrototypeOf,u=Object.prototype.hasOwnProperty;var s=o=>a(o,"__esModule",{value:!0});var m=(o,e)=>()=>(e||o((e={exports:{}}).exports,e),e.exports),f=(o,e)=>{s(o);for(var n in e)a(o,n,{get:e[n],enumerable:!0})},g=(o,e,n)=>{if(e&&typeof e=="object"||typeof e=="function")for(let i of d(e))!u.call(o,i)&&i!=="default"&&a(o,i,{get:()=>e[i],enumerable:!(n=h(e,i))||n.enumerable});return o},w=o=>g(s(a(o!=null?c(p(o)):{},"default",o&&o.__esModule&&"default"in o?{get:()=>o.default,enumerable:!0}:{value:o,enumerable:!0})),o);var l=m((I,r)=>{r.exports=_jsx_runtime});var k={};f(k,{default:()=>v,frontmatter:()=>y});var t=w(l()),y={title:"Infrastructure as Code for Production Ops",date:new Date(1631718043415),author:"Charles Cary",featuredImage:"/uploads/posts/istock-1224441051-copy.jpg",excerpt:`DevOps leaders can apply infrastructure as code lessons and tooling to production ops, use solutions like Terraform + Shoreline to automate repeatable tasks, and make hero-level institutional knowledge accessible to anyone.
`};function b(o){let e=Object.assign({p:"p",em:"em",a:"a",h2:"h2",ol:"ol",li:"li"},o.components),{wrapper:n}=e,i=(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(e.p,{children:(0,t.jsx)(e.em,{children:"Summary: DevOps thrives in the face of the unexpected. People working in the industry must be able to respond to unplanned incidents with expertise, quality, and efficiency. This post details how DevOps leaders can apply infrastructure as code lessons and tooling to production ops, use solutions like Terraform + Shoreline to automate repeatable tasks, and make hero-level institutional knowledge accessible to anyone."})}),`
`,(0,t.jsx)(e.p,{children:"I\u2019ve been a hero in ops. You know the type: jumps on tickets in minutes and spends hours on a system-critical fix for a specific customer. During my time in the role, I handled hundreds of tickets and fixed some of the biggest IT challenges ever faced in cloud."}),`
`,(0,t.jsx)(e.p,{children:"I\u2019ll be honest, it was an adrenaline rush\u2014especially when time was limited and success required making the smallest action to quite literally save the day (think: selecting the right wire to cut in an action movie).\xA0"}),`
`,(0,t.jsxs)(e.p,{children:["But there was a point when the rush became a slog. All of that repeated work was really ",(0,t.jsx)(e.a,{href:"https://cloud.google.com/blog/products/management-tools/identifying-and-tracking-toil-using-sre-principles",children:"toil"}),". It didn't accumulate or scale out. Critical issues turned into constant problems because manual ops isn\u2019t like software development. If you fix something once, you\u2019ll definitely need to do it again (and again)\u2014likely the next day."]}),`
`,(0,t.jsx)(e.h2,{id:"where-we-are-today",children:"Where we are today"}),`
`,(0,t.jsxs)(e.p,{children:["These days, a fair number of teams have been able to limit the need for heroism and replace infrastructure tickets with ",(0,t.jsx)(e.a,{href:"https://docs.microsoft.com/en-us/devops/deliver/what-is-infrastructure-as-code",children:"infrastructure as code"}),"."]}),`
`,(0,t.jsxs)(e.p,{children:["The next phase is ",(0,t.jsx)(e.a,{href:"https://registry.terraform.io/providers/shorelinesoftware/shoreline/latest",children:"infrastructure as code for production ops"}),", or moving the practices of operations\u2014the runbooks, playbooks, scripts, and other knowhow\u2014out of wikis or operators\u2019 heads and into code (just like we did with infrastructure and software before it). This is the key building block for moving toward an automated future where people with any level of expertise will be able to orchestrate operations flawlessly."]}),`
`,(0,t.jsxs)(e.p,{children:["In simple terms, infrastructure as code for production ops offers a way to make actions repeatable and automate work. In practice, it involves managing ops using similar tooling (alarms, remediations, checks, tests, verifications) that one does for managing the code underlying software. The key benefit here is that, when applied correctly, an ops team member just needs to implement a code change and the infrastructure as code ",(0,t.jsx)(e.a,{href:"https://github.com/shorelinesoftware/terraform-provider-shoreline",children:"tool"})," or architecture does the rest."]}),`
`,(0,t.jsx)(e.h2,{id:"isnt-the-answer-kubernetes",children:"Isn\u2019t the answer Kubernetes?"}),`
`,(0,t.jsx)(e.p,{children:"If you\u2019re a SRE, you might be thinking: wait a minute, the answer here is Kubernetes.\xA0"}),`
`,(0,t.jsx)(e.p,{children:"It's true that Kubernetes does handle scheduling of pods on our nodes (i.e. solves the problem of maximizing resource utilization). For example, we still set up monitoring tools (e.g. Prometheus) and still have runbooks / playbooks on Kubernetes. Sometimes, we manually restart pods, adjust disk size, and handle poor application performance. But, that's not where the work of ops teams ends.\xA0"}),`
`,(0,t.jsx)(e.p,{children:"It\u2019s important to remember that ops is fundamentally about the unexpected: we know that we need people to handle incidents that arise, but aren't planned. The truth is that no deployment tool solves this problem because they are for regular (planned) rollout of software.\xA0"}),`
`,(0,t.jsx)(e.h2,{id:"a-closer-look-at-infrastructure-as-code-for-production-ops",children:"A closer look at infrastructure as code for production ops"}),`
`,(0,t.jsx)(e.p,{children:"We have the automation, architecture, and expertise to apply it to DevOps/SRE work at scale. But the lessons from infrastructure as code are getting dropped as soon as projects enter operations and production phases.\xA0"}),`
`,(0,t.jsxs)(e.p,{children:["In practice, we need to extend infrastructure as code to ops by employing specific tools, like a ",(0,t.jsx)(e.a,{href:"https://github.com/shorelinesoftware/terraform-provider-shoreline",children:"Terraform"})," provider, to deliver version control, code review, testing, and deployment all in one place throughout remediations\u2014the runbooks of ops or the steps needed to get things fixed.\xA0"]}),`
`,(0,t.jsx)(e.p,{children:"We need to consider doing this across the board, but especially in modern environments where folks are using things like containers (constantly coming and going) and autoscale virtual machines (constantly scaling up or down dynamically). It also matters significantly in data pipeline, storage, and processing environments\u2014all of which tend to be stateful i.e. operational pain is acute.\xA0"}),`
`,(0,t.jsx)(e.h2,{id:"a-collective-memory-for-devops--sre",children:"A 'collective memory' for DevOps & SRE"}),`
`,(0,t.jsx)(e.p,{children:"Another pain point in the path of efficiency and customer satisfaction is that individual ops team members can\u2019t handle incidents consistently without collective memory. Put another way, different folks come up with their own way of solving things, and doing so with varying degrees of efficacy and safety. This can create issues for managers and team leads, including:"}),`
`,(0,t.jsxs)(e.ol,{children:[`
`,(0,t.jsx)(e.li,{children:"Stress about who might be oncall because there is such a difference in quality between manual work conducted by senior and junior folks."}),`
`,(0,t.jsx)(e.li,{children:"A higher risk of people doing the wrong thing because they are looking at outdated runbooks / playbooks."}),`
`,(0,t.jsx)(e.li,{children:"Failure to get as many people involved in operations as possible despite a company effort to decentralize DevOps (where engineers own their code and operations)\u2014the all-hands-on-deck approach simply can\u2019t work if you can't propagate information."}),`
`]}),`
`,(0,t.jsx)(e.p,{children:"But it doesn\u2019t have to be like that.\xA0"}),`
`,(0,t.jsx)(e.h2,{id:"how-to-achieve-ops-team-success-in-fast-paced-environments",children:"How to achieve ops team success in fast-paced environments"}),`
`,(0,t.jsx)(e.p,{children:"Ops team success requires achieving availability of applications in the face of rapidly scaling environments and infrastructure. Every machine, piece of software, and infrastructure asset has a chance of failure. When you have a small number of these things, that chance is miniscule most of the time. But, as you scale out, that small chance starts multiplying. Eventually, at scale, you are having unplanned incidents continuously. This is where automation is so key.\xA0"}),`
`,(0,t.jsx)(e.p,{children:"Not to mention that DevOps/SRE mistakes directly impact specific SLAs and overall customer experiences\u2013especially in cases where fleets grow much faster than ops team headcount. Applying infrastructure as code for production ops can help convert repetitive, time-intensive tasks into work that\u2019s completed once and applied from the initial fix onward through automation. This, in turn, can relieve headcount pressure and reduce mistakes or disruptions that reflect poorly on ops and, frankly, an entire organization.\xA0"}),`
`,(0,t.jsx)(e.h2,{id:"potential-hurdles-of-infrastructure-as-code-for-production-ops",children:"Potential hurdles of infrastructure as code for production ops\xA0"}),`
`,(0,t.jsxs)(e.p,{children:["One of the primary challenges of applying infrastructure as code for production ops to modern IT environments is that automation requires significant time investment along with the initial financial investment. There\u2019s a high resource barrier to entry for a lot of companies that would otherwise subscribe to infrastructure as code for production ops. Shoreline\u2019s ",(0,t.jsx)(e.a,{href:"https://shoreline.io/demo",children:"product"})," addresses this head-on by reducing the time it takes to automate tasks which, in turn, allows customers to produce more while paying less."]}),`
`,(0,t.jsx)(e.p,{children:"Another issue at the core of ops: incidents are unplanned and the chance they occur multiplies along with the number of machines, software solutions, etc. added to the mix as an IT operation scales. For instance, most mid-sized companies have not one Kubernetes cluster, but dozens. Most have multiple servers. So, the question becomes: how do you scale ops to manage all of that?"}),`
`,(0,t.jsx)(e.h2,{id:"making-sense-of-collective-memory-for-production-ops",children:"Making sense of \u2018collective memory\u2019 for production ops"}),`
`,(0,t.jsx)(e.p,{children:"DevOps/SRE staff build up a lot of specific knowledge of their craft. For example, they memorize the specific sequence to use when restarting a system. It\u2019s common nowadays for this knowledge today in runbooks stored in wikis. However, these are not living documents\u2014they often become stale and inaccurate. Additionally, knowledge trapped in the heads of DevOps/SRE staff is lost to the rest of the organization."}),`
`,(0,t.jsxs)(e.p,{children:['With Shoreline, operators can store the resources, metrics, and actions they know of in a "',(0,t.jsx)(e.a,{href:"https://shoreline.io/blog/why-systems-fail-and-what-you-can-do-about-it",children:"collective memory"}),'"\u2014the symbols can then be shared between the operators and updated as needed. Since the definitions are shared, rather than trapped in folks\u2019 heads, everyone is on the same page and nothing gets lost. For example, a junior engineer can describe a symbol to gain the insight a senior engineer has added to Shoreline. Support personnel can handle run actions to help customers (with the confidence they are doing them correctly) in line with current best practices at the company.']}),`
`,(0,t.jsx)(e.h2,{id:"one-step-further-how-composability-enables-efficiency",children:"One step further: how composability enables efficiency\xA0"}),`
`,(0,t.jsxs)(e.p,{children:["Furthermore, these symbols are composable. Shoreline's ",(0,t.jsx)(e.a,{href:"https://docs.shoreline.io/op/tutorial",children:"Op language"})," and system allow for the combination of resources, metrics, and actions to ensure that every symbol strengthens the others. In other words, the act of someone adding a new symbol increases the collective group of operators\u2019 ability to combine workflows and solve problems."]}),`
`,(0,t.jsxs)(e.p,{children:["DevOps/SRE are incentivized to use the system because symbols are built into the tool rather than putting the information on a wiki that no one knows how to access. Instead, DevOps/SRE are using the ",(0,t.jsx)(e.a,{href:"https://softwareengineeringdaily.com/2021/07/29/shoreline-fleet-automation-with-anurag-gupta/",children:"Shoreline CLI"})," to actually solve the issues."]}),`
`,(0,t.jsx)(e.h2,{id:"looking-back-to-go-forward",children:"Looking back to go forward\xA0"}),`
`,(0,t.jsx)(e.p,{children:(0,t.jsxs)(e.em,{children:["DevOps is a critical, customer-facing function. By applying infrastructure as code lessons to production ops, managers and team leads can employ specific tools, like ",(0,t.jsx)(e.a,{href:"https://github.com/shorelinesoftware/terraform-provider-shoreline",children:"Shoreline's Terraform provider"}),", to automate tasks or remediations, and tap into the knowledge bank provided by ",(0,t.jsx)(e.a,{href:"https://softwareengineeringdaily.com/2021/07/29/shoreline-fleet-automation-with-anurag-gupta/",children:"Shoreline CLI"})," to ensure quality incident response every time."]})}),`
`,(0,t.jsx)(e.p,{children:"I remember writing code before it was consistently applied. In those days, if someone threw out the code or messed with it in some way, we were like: whelp, it\u2019s time to start all over.\xA0"}),`
`,(0,t.jsx)(e.p,{children:"That may sound ridiculous today, but it was the state of the software world at one point. It was the state of infrastructure until infrastructure as code came along."}),`
`,(0,t.jsx)(e.p,{children:"Now, we\u2019re kind of at a similar point in ops. An ops person who completes a remediation one day won\u2019t solve that issue from now on\u2014on the contrary, the work required to address it manually is bound to be repeated over and over for as long as that person is in the job.\xA0"}),`
`,(0,t.jsx)(e.p,{children:"The technology and underlying coding principles exist to eliminate repetitive work and automate context-specific remediations. At Shoreline, we want to go even further to make ops, including oncall, accessible to the entire team. We believe embracing infrastructure as code for production ops will get us there."}),`
`,(0,t.jsxs)(e.p,{children:["To see how Shoreline partners with Terraform to extend infrastructure as code to production ops, ",(0,t.jsx)(e.a,{href:"https://shoreline.io/demo",children:"request a demo"}),"."]})]});return n?(0,t.jsx)(n,Object.assign({},o,{children:i})):i}var v=b;return k;})();
;return Component.default;

Infrastructure as Code for Production Ops

var Component=(()=>{var c=Object.create;var o=Object.defineProperty;var a=Object.getOwnPropertyDescriptor;var d=Object.getOwnPropertyNames;var p=Object.getPrototypeOf,m=Object.prototype.hasOwnProperty;var i=t=>o(t,"__esModule",{value:!0});var u=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports),A=(t,e)=>{i(t);for(var s in e)o(t,s,{get:e[s],enumerable:!0})},E=(t,e,s)=>{if(e&&typeof e=="object"||typeof e=="function")for(let r of d(e))!m.call(t,r)&&r!=="default"&&o(t,r,{get:()=>e[r],enumerable:!(s=a(e,r))||s.enumerable});return t},v=t=>E(i(o(t!=null?c(p(t)):{},"default",t&&t.__esModule&&"default"in t?{get:()=>t.default,enumerable:!0}:{value:t,enumerable:!0})),t);var l=u((w,h)=>{h.exports=_jsx_runtime});var T={};A(T,{default:()=>_,frontmatter:()=>f});var n=v(l()),f={title:"Analyze and act upon remediation incidents with Shoreline Events",date:new Date(1631275453037),author:"Gabe Wyatt",featuredImage:"/uploads/posts/events.png",excerpt:"Learn how to filter, sort, analyze, and act upon remediation incidents with Shoreline Events.",slug:"analyze-act-remediation-incidents-with-shoreline-events"};function R(t){let e=Object.assign({p:"p",a:"a",h2:"h2",ul:"ul",li:"li",code:"code",pre:"pre",hr:"hr"},t.components),{wrapper:s}=e,r=(0,n.jsxs)(n.Fragment,{children:[(0,n.jsxs)(e.p,{children:["Shoreline makes it easy to automate fleet-wide remediation by creating an ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/alarms",children:"Alarm"})," associated with your infrastructure ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/platform/resources",children:"Resources"})," -- the ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/alarms",children:"Alarm"})," fires when an incident occurs, which executes an associated remediation ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/actions",children:"Action"}),". Monitoring everything Shoreline does during this remediation process is accessible via the graphical ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/ui/monitoring",children:"web UI"}),". Still, we knew it was critical to provide access to the same data via the CLI."]}),`
`,(0,n.jsxs)(e.p,{children:["We're excited to announce the release of ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events",children:"Shoreline Events"}),", which allows you to use the ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/installation#cli",children:"Op CLI"})," to evaluate the status of every ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/alarms",children:"Alarm"}),", ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/actions",children:"Action"}),", ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/bots",children:"Bot"}),", and ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/platform/resources",children:"Resource"})," within your infrastructure. Every part of the remediation loop is tracked and associated with an Event, giving you the ability to execute many ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/commands",children:"Op commands"})," to filter and act upon the results."]}),`
`,(0,n.jsx)(e.h2,{id:"event-types",children:"Event Types"}),`
`,(0,n.jsx)(e.p,{children:"Shoreline generates Events anytime a core object is created or modified."}),`
`,(0,n.jsxs)(e.ul,{children:[`
`,(0,n.jsxs)(e.li,{children:[(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#alarm-events",children:"Alarm Events"})," - Created whenever an ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/alarms",children:"Alarm"})," triggers, resolves, or ends."]}),`
`,(0,n.jsxs)(e.li,{children:[(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#action-events",children:"Action Events"})," - Created whenever an ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/actions",children:"Action"})," executes, completes, or fails."]}),`
`,(0,n.jsxs)(e.li,{children:[(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#bot-events",children:"Bot Events"})," - Created whenever a ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/bots",children:"Bot"})," executes, completes, or fails."]}),`
`,(0,n.jsxs)(e.li,{children:[(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#resource-events",children:"Resource Events"})," - Created whenever a ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/platform/resources",children:"Resource"})," registers or de-registers."]}),`
`]}),`
`,(0,n.jsxs)(e.p,{children:["For example, use the ",(0,n.jsx)(e.code,{children:'events | type="alarm"'})," command to get all ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#alarm-events",children:"Alarm Events"}),":"]}),`
`,(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{children:`op> events | type="alarm"

RESOURCE_NAME              | RESOURCE_TYPE | ALARM_NAME                           | STATUS    | STEP_TYPE   | TIMESTAMP
i-0192cd8c4f55219c2        | HOST          | JSON_Metric_auto_name                | triggered |             |
                           |               |                                      |           | ALARM_FIRE  | 2021-08-31T14:47:20-07:00
                           |               |                                      |           | ALARM_CLEAR | 1969-12-31T16:00:00-08:00
test2-cust.shoreline-l2jf7 | POD           | simple_pod_alarm                     | resolved  |             |
                           |               |                                      |           | ALARM_FIRE  | 2021-08-30T11:05:44-07:00
                           |               |                                      |           | ALARM_CLEAR | 2021-08-30T11:06:02-07:00
test2-cust.shoreline-l2jf7 | POD           | mixed_metric_linux_cmd_alarm         | resolved  |             |
                           |               |                                      |           | ALARM_FIRE  | 2021-08-30T11:05:44-07:00
                           |               |                                      |           | ALARM_CLEAR | 2021-08-30T11:06:02-07:00
test2-cust.shoreline-l2jf7 | POD           | linux_cmd_duration_res_env_var_alarm | resolved  |             |
                           |               |                                      |           | ALARM_FIRE  | 2021-08-30T11:05:44-07:00
                           |               |                                      |           | ALARM_CLEAR | 2021-08-30T11:06:02-07:00
i-00f4a29cc5a8716d1        | HOST          | always_fire_alarm                    | resolved  |             |
                           |               |                                      |           | ALARM_FIRE  | 2021-08-30T11:05:43-07:00
                           |               |                                      |           | ALARM_CLEAR | 2021-08-30T11:05:52-07:00
`})}),`
`,(0,n.jsx)(e.h2,{id:"filter-events",children:"Filter Events"}),`
`,(0,n.jsxs)(e.p,{children:["You can ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#filter-events",children:"filter Event records"})," using many of the same ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/commands",children:"Op commands"})," that power the rest of the platform. Check out some simple examples below."]}),`
`,(0,n.jsxs)(e.ul,{children:[`
`,(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#count",children:"Count the number of Alarm Events"})}),`
`]}),`
`,(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{children:`op> events | type="alarm" | count

RESOURCE_TYPE | RESOURCE_NAME                        | EVENT_TYPE | FIRED | CLEARED | ACTIVE | TOTAL_COUNT
              |                                      | ALARMS     | 54    | 54      | 0      | 54
              |                                      | ALARMS     | 16    | 16      | 0      | 16
POD           | test5-cust.shoreline-nvbdm           |            |       |         |        |
              |                                      | ALARMS     | 1     | 1       | 0      | 1
CONTAINER     | test5-cust.shoreline-729bc.shoreline |            |       |         |        |
              |                                      | ALARMS     | 16    | 16      | 0      | 16
POD           | test5-cust.shoreline-729bc           |            |       |         |        |
              |                                      | ALARMS     | 1     | 1       | 0      | 1
HOST          | i-0d991c45ccdb96db4                  |            |       |         |        |
              |                                      | ALARMS     | 1     | 1       | 0      | 1
CONTAINER     | test5-cust.shoreline-wtjfd.shoreline |            |       |         |        |
              |                                      | ALARMS     | 16    | 16      | 0      | 16
POD           | test5-cust.shoreline-wtjfd           |            |       |         |        |
              |                                      | ALARMS     | 3     | 3       | 0      | 3
CONTAINER     | test5-cust.shoreline-nvbdm.shoreline |            |       |         |        |
`})}),`
`,(0,n.jsxs)(e.ul,{children:[`
`,(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#status",children:"Get Events by status"})}),`
`]}),`
`,(0,n.jsxs)(e.p,{children:["For example, this command gets the first five ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#alarm-events",children:"Alarm Events"})," that fired and then resolved:"]}),`
`,(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{children:`op> events | type="alarm" | status="resolved" | limit=5

RESOURCE_NAME              | RESOURCE_TYPE | ALARM_NAME                     | STATUS   | STEP_TYPE   | TIMESTAMP
test4-cust.shoreline-9qm7j | POD           | simple_pod_alarm               | resolved |             |
                           |               |                                |          | ALARM_FIRE  | 2021-09-01T15:44:07-07:00
                           |               |                                |          | ALARM_CLEAR | 2021-09-01T15:44:36-07:00
test4-cust.shoreline-9qm7j | POD           | mixed_metric_linux_cmd_alarm   | resolved |             |
                           |               |                                |          | ALARM_FIRE  | 2021-09-01T15:43:07-07:00
                           |               |                                |          | ALARM_CLEAR | 2021-09-01T15:44:36-07:00
test4-cust.shoreline-9qm7j | POD           | linux_cmd_duration_res_env_var | resolved |             |
                           |               |                                |          | ALARM_FIRE  | 2021-09-01T15:43:07-07:00
                           |               |                                |          | ALARM_CLEAR | 2021-09-01T15:44:36-07:00
test4-cust.shoreline-zrwhf | POD           | mixed_metric_linux_cmd_alarm   | resolved |             |
                           |               |                                |          | ALARM_FIRE  | 2021-09-01T15:43:00-07:00
                           |               |                                |          | ALARM_CLEAR | 2021-09-01T15:43:18-07:00
test4-cust.shoreline-zrwhf | POD           | linux_cmd_duration_res_env_var | resolved |             |
                           |               |                                |          | ALARM_FIRE  | 2021-09-01T15:43:00-07:00
                           |               |                                |          | ALARM_CLEAR | 2021-09-01T15:43:18-07:00
`})}),`
`,(0,n.jsxs)(e.ul,{children:[`
`,(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#by-resource",children:"Get Events associated with specific Resources"})}),`
`]}),`
`,(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{children:`op> pods | name=~"shoreline-mxxhw" | events | type="bot" | count

RESOURCE_NAME              | EVENT_TYPE | TRIGGERED | FINISHED | ACTIVE | TOTAL_COUNT
                           | BOTS       | 18        | 18       | 0      | 18
                           | BOTS       | 18        | 18       | 0      | 18
test4-cust.shoreline-mxxhw |            |           |          |        |
`})}),`
`,(0,n.jsxs)(e.ul,{children:[`
`,(0,n.jsx)(e.li,{children:(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#by-column-or-tag",children:"Get Events by tags"})}),`
`]}),`
`,(0,n.jsxs)(e.p,{children:["For example, the following finds ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#alarm-events",children:"Alarm Events"})," for ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/platform/resources",children:"Resources"})," within either the ",(0,n.jsx)(e.code,{children:"us-west-2a"})," or ",(0,n.jsx)(e.code,{children:"us-west-2b"})," availability zones."]}),`
`,(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{children:`op> events | type="alarm" | az=["us-west-2a", "us-west-2b"] | count

RESOURCE_TYPE | RESOURCE_NAME       | AZ         | EVENT_TYPE | FIRED | CLEARED | ACTIVE | TOTAL_COUNT
              |                     |            | ALARMS     | 99    | 93      | 6      | 99
              |                     |            | ALARMS     | 4     | 3       | 1      | 4
HOST          | i-0777445c24c2b8344 | us-west-2b |            |       |         |        |
              |                     |            | ALARMS     | 5     | 4       | 1      | 5
HOST          | i-0627f066048c1b3d9 | us-west-2a |            |       |         |        |
              |                     |            | ALARMS     | 6     | 5       | 1      | 6
HOST          | i-00f4a29cc5a8716d1 | us-west-2b |            |       |         |        |
              |                     |            | ALARMS     | 4     | 3       | 1      | 4
HOST          | i-0192cd8c4f55219c2 | us-west-2a |            |       |         |        |
`})}),`
`,(0,n.jsxs)(e.p,{children:["Check out the ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events#filter-events",children:"Filter Events documentation"})," for more details."]}),`
`,(0,n.jsx)(e.h2,{id:"filter-resources-by-events",children:"Filter Resources by Events"}),`
`,(0,n.jsxs)(e.p,{children:["Alter the Resources returned by a ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/platform/resources#resource-relationships",children:"Resource query"})," based on associated Events. This capability is particularly potent when piped to additional ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/commands",children:"Op commands"}),"."]}),`
`,(0,n.jsx)(e.p,{children:"To illustrate, consider the following Op statement:"}),`
`,(0,n.jsx)(e.pre,{children:(0,n.jsx)(e.code,{children:'op> hosts | filter((events | type="alarm" | name="my_cpu_alarm" | status="triggered" | count) >= 3) | `top -b -n 1 | head -n 15`\n'})}),`
`,(0,n.jsx)(e.p,{children:"This statement goes through the following steps:"}),`
`,(0,n.jsxs)(e.ul,{children:[`
`,(0,n.jsx)(e.li,{children:"Get all host Resources"}),`
`,(0,n.jsxs)(e.li,{children:["Filter down hosts to those that ",(0,n.jsx)(e.code,{children:"triggered"})," the ",(0,n.jsx)(e.code,{children:"my_cpu_alarm"})," Alarm at least three times"]}),`
`,(0,n.jsx)(e.li,{children:"Execute a custom Linux command to output the current processes"}),`
`]}),`
`,(0,n.jsx)(e.hr,{}),`
`,(0,n.jsx)(e.p,{children:"Although we've just scratched the surface of what it can do, the new Events command allows you to use the CLI to determine the status of everything that Shoreline is remediating on your behalf."}),`
`,(0,n.jsxs)(e.p,{children:["Head over to the ",(0,n.jsx)(e.a,{href:"https://docs.shoreline.io/op/events",children:"Events documentation"})," for more details and examples. If you're ready to jump in and see what Shoreline can do, please ",(0,n.jsx)(e.a,{href:"https://shoreline.io/demo?ref=blog&topic=Events",children:"click here"})," to request a demo."]})]});return s?(0,n.jsx)(s,Object.assign({},t,{children:r})):r}var _=R;return T;})();
;return Component.default;

Analyze and act upon remediation incidents with Shoreline Events

var Component=(()=>{var u=Object.create;var s=Object.defineProperty;var g=Object.getOwnPropertyDescriptor;var m=Object.getOwnPropertyNames;var p=Object.getPrototypeOf,h=Object.prototype.hasOwnProperty;var r=e=>s(e,"__esModule",{value:!0});var d=(e,o)=>()=>(o||e((o={exports:{}}).exports,o),o.exports),f=(e,o)=>{r(e);for(var n in o)s(e,n,{get:o[n],enumerable:!0})},x=(e,o,n)=>{if(o&&typeof o=="object"||typeof o=="function")for(let a of m(o))!h.call(e,a)&&a!=="default"&&s(e,a,{get:()=>o[a],enumerable:!(n=g(o,a))||n.enumerable});return e},y=e=>x(r(s(e!=null?u(p(e)):{},"default",e&&e.__esModule&&"default"in e?{get:()=>e.default,enumerable:!0}:{value:e,enumerable:!0})),e);var l=d((M,c)=>{c.exports=_jsx_runtime});var _={};f(_,{default:()=>j,frontmatter:()=>b});var t=y(l()),b={title:"Using Shoreline.io to root-cause transient issues (like JVM garbage collection)",date:new Date(1627379529953),author:"Charles Cary",featuredImage:"/uploads/posts/java-garbage-collection.jpg",excerpt:"Shoreline makes it easy to collect diagnostic information when you're doing a root-cause analysis of an issue. This example shows how to automatically capture debugging information for slow Java garbage collection and then automatically bounce the process to alleviate customer pain."};function w(e){let o=Object.assign({p:"p"},e.components),{YouTube:n,wrapper:a}=o,i=(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n,{id:"1UM4qw5rF_s"}),`
`,(0,t.jsx)(o.p,{children:"Shoreline makes it easy to collect diagnostic information when you're doing a root-cause analysis of an issue. This example shows how to automatically capture debugging information for slow Java garbage collection and then automatically bounce the process to alleviate customer pain."})]});return a?(0,t.jsx)(a,Object.assign({},e,{children:i})):i}var j=w;return _;})();
;return Component.default;

Using Shoreline.io to root-cause transient issues (like JVM garbage collection)

Ready to get started?