Skip to content

Commit f6d3a46

Browse files
add a 'panic' button
- in the real world things just fail often unexpectedly and due to reasons that one will never fully understand. In order to cope with this circunstances and be fully prepared for them - because they will happen corectl now has a "panic" option that will hard kill a given running VM (or a randomly picked one). This feature is intended as a practical way to test and reproduce both cluster failure scenarios and the layouts resilient to them - en passant the stock the 'halt' option now complains when tasked to halt stuff that doesn't exist Signed-off-by: António Meireles <antonio.meireles@reformi.st>
1 parent 2e040f4 commit f6d3a46

9 files changed

Lines changed: 205 additions & 22 deletions

File tree

‎cmd/client/halt.go‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,14 @@ var (
3030
Short: "Halts one or more running CoreOS instances",
3131
PreRunE: func(cmd *cobra.Command, args []string) (err error) {
3232
session.Caller.CmdLine.BindPFlags(cmd.Flags())
33-
if len(args) < 1 && !session.Caller.CmdLine.GetBool("all") {
34-
err = fmt.Errorf("This command requires either at least " +
35-
"one argument to work or --all.")
33+
errMsg := fmt.Errorf("This command requires either " +
34+
"one argument to work or just '--all'.")
35+
if session.Caller.CmdLine.GetBool("all") {
36+
if len(args) != 0 {
37+
err = errMsg
38+
}
39+
} else if len(args) != 1 {
40+
err = errMsg
3641
}
3742
return
3843
},
@@ -41,15 +46,16 @@ var (
4146
)
4247

4348
func killCommand(cmd *cobra.Command, args []string) (err error) {
49+
var in server.RPCquery
50+
4451
if _, err = server.Daemon.Running(); err != nil {
4552
return session.ErrServerUnreachable
4653
}
4754

48-
if session.Caller.CmdLine.GetBool("all") {
49-
_, err = server.RPCQuery("StopVMs", &server.RPCquery{})
50-
} else {
51-
_, err = server.RPCQuery("StopVMs", &server.RPCquery{Input: args})
55+
if !session.Caller.CmdLine.GetBool("all") {
56+
in.Input = args
5257
}
58+
_, err = server.RPCQuery("StopVMs", &in)
5359
return
5460
}
5561

‎cmd/client/panic.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright (c) 2016 by António Meireles <antonio.meireles@reformi.st>.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
16+
package main
17+
18+
import (
19+
"fmt"
20+
21+
"github.com/TheNewNormal/corectl/components/host/session"
22+
"github.com/TheNewNormal/corectl/components/server"
23+
"github.com/deis/pkg/log"
24+
"github.com/spf13/cobra"
25+
)
26+
27+
var (
28+
panicCmd = &cobra.Command{
29+
Use: "panic [VMids]",
30+
Aliases: []string{"oops", "wtf"},
31+
Short: "Hard kills a running CoreOS instance",
32+
Long: "Hard kills a running CoreOS instance.\nThis feature is " +
33+
"intended as a practical way to test and reproduce both cluster " +
34+
"failure scenarios and layouts resilient to them",
35+
PreRunE: func(cmd *cobra.Command, args []string) (err error) {
36+
session.Caller.CmdLine.BindPFlags(cmd.Flags())
37+
errMsg := fmt.Errorf("This command requires either " +
38+
"one argument to work or just '--random'.")
39+
if session.Caller.CmdLine.GetBool("random") {
40+
if len(args) != 0 {
41+
err = errMsg
42+
}
43+
} else if len(args) != 1 {
44+
err = errMsg
45+
}
46+
return
47+
},
48+
RunE: panicCommand,
49+
}
50+
)
51+
52+
func panicCommand(cmd *cobra.Command, args []string) (err error) {
53+
var (
54+
in server.RPCquery
55+
out *server.RPCreply
56+
random = session.Caller.CmdLine.GetBool("random")
57+
)
58+
59+
if _, err = server.Daemon.Running(); err != nil {
60+
return session.ErrServerUnreachable
61+
}
62+
63+
if !random {
64+
in.Input = args
65+
}
66+
in.Forced = true
67+
if out, err = server.RPCQuery("StopVMs", &in); err != nil {
68+
return
69+
}
70+
if random {
71+
log.Info("'%v' gone", out.Output[0])
72+
}
73+
return
74+
}
75+
76+
func init() {
77+
panicCmd.Flags().BoolP("random", "r", false,
78+
"hard kill a randomly choosen running instance")
79+
rootCmd.AddCommand(panicCmd)
80+
}

‎components/server/rpcservices.go‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"bytes"
2121
"fmt"
2222
"io"
23+
"math/rand"
2324
"net/http"
2425
"os/exec"
2526
"path/filepath"
@@ -41,15 +42,18 @@ import (
4142

4243
var (
4344
rpcServices = rpc.NewServer()
44-
ErrServerShuttingDown = fmt.Errorf("Request ignored as server shutting down")
45+
ErrServerShuttingDown = fmt.Errorf("Request ignored as server is shutting down")
46+
ErrNothingToShutdown = fmt.Errorf("Request ignored as no VMs were found running")
47+
ErrUnknownVM = fmt.Errorf("Request ignored as no VM with requested name or UUID was found")
4548
)
4649

4750
type (
4851
RPCservice struct{}
4952

5053
RPCquery struct {
51-
Input []string
52-
VM *VMInfo
54+
Input []string
55+
Forced bool
56+
VM *VMInfo
5357
}
5458
RPCreply struct {
5559
Output []string
@@ -254,7 +258,7 @@ func (s *RPCservice) Run(r *http.Request,
254258
select {
255259
case <-timeout:
256260
vm.Pid = vm.exec.Process.Pid
257-
vm.halt()
261+
vm.gracefullyShutdown()
258262
vm.errCh <- fmt.Errorf("Unable to grab VM's IP after " +
259263
"30s (!)... Aborted")
260264
case ip := <-vm.publicIPCh:
@@ -316,7 +320,7 @@ func (s *RPCservice) Stop(r *http.Request,
316320
Daemon.AcceptingRequests = false
317321
Daemon.Unlock()
318322

319-
Daemon.Active.array().halt()
323+
Daemon.Active.array().gracefullyShutdown()
320324
Daemon.Oops <- nil
321325
return
322326
}
@@ -344,7 +348,17 @@ func (s *RPCservice) StopVMs(r *http.Request,
344348
}
345349

346350
if len(args.Input) == 0 {
347-
targets = Daemon.Active.array()
351+
active := Daemon.Active.array()
352+
if len(active) == 0 {
353+
return ErrNothingToShutdown
354+
}
355+
if !args.Forced {
356+
targets = active
357+
} else {
358+
// random pick
359+
targets = append(targets, active[rand.Intn(len(active))])
360+
reply.Output = append(reply.Output, targets[0].Name)
361+
}
348362
} else {
349363
for _, t := range args.Input {
350364
for _, v := range Daemon.Active {
@@ -353,8 +367,16 @@ func (s *RPCservice) StopVMs(r *http.Request,
353367
}
354368
}
355369
}
370+
if len(targets) != len(args.Input) {
371+
return ErrUnknownVM
372+
}
356373
}
357-
targets.halt()
374+
if !args.Forced {
375+
targets.gracefullyShutdown()
376+
} else {
377+
targets[0].kill()
378+
}
379+
358380
return
359381
}
360382

‎components/server/run.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -338,12 +338,12 @@ func (vm *VMInfo) assembleBootPayload() (xArgs []string, err error) {
338338
err
339339
}
340340

341-
func (list VMs) halt() {
341+
func (list VMs) gracefullyShutdown() {
342342
sort.Sort(sort.Reverse(VMs(list)))
343343

344344
for _, v := range list {
345345
log.Info("shutting down %v...", v.Name)
346-
Daemon.Active[v.UUID].halt()
346+
Daemon.Active[v.UUID].gracefullyShutdown()
347347
for {
348348
Daemon.Lock()
349349
_, stillAlive := Daemon.Active[v.UUID]
@@ -357,7 +357,14 @@ func (list VMs) halt() {
357357
return
358358
}
359359

360-
func (vm *VMInfo) halt() {
360+
func (vm *VMInfo) kill() {
361+
log.Debug("hard killing %v", vm.Name)
362+
if err := vm.exec.Process.Kill(); err != nil {
363+
log.Err(err.Error())
364+
}
365+
}
366+
367+
func (vm *VMInfo) gracefullyShutdown() {
361368
// Try to gracefully terminate the process.
362369
if err := vm.exec.Process.Signal(syscall.SIGTERM); err != nil {
363370
log.Err(err.Error())
@@ -366,9 +373,7 @@ func (vm *VMInfo) halt() {
366373
case <-time.After(ServerTimeout):
367374
log.Err("Attempting to halt %v (%v) with SIGTERM timed out, "+
368375
"SIGKILLing it now.", vm.UUID, vm.exec.Process.Pid)
369-
if err := vm.exec.Process.Kill(); err != nil {
370-
log.Err(err.Error())
371-
}
376+
vm.kill()
372377
case <-vm.done:
373378
}
374379
}

‎components/server/server.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func Start() (err error) {
124124
log.Info("Got '%v' signal, stopping server...", s)
125125
signal.Stop(hades)
126126
Daemon.Oops <- nil
127-
Daemon.Active.array().halt()
127+
Daemon.Active.array().gracefullyShutdown()
128128
}()
129129

130130
log.Info("server starting...")

‎documentation/man/corectl.1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ Copyright (c) 2015\-2016, António Meireles
2828

2929
.SH SEE ALSO
3030
.PP
31-
\fBcorectl\-kill(1)\fP, \fBcorectl\-load(1)\fP, \fBcorectl\-ls(1)\fP, \fBcorectl\-ps(1)\fP, \fBcorectl\-pull(1)\fP, \fBcorectl\-put(1)\fP, \fBcorectl\-query(1)\fP, \fBcorectl\-rm(1)\fP, \fBcorectl\-run(1)\fP, \fBcorectl\-ssh(1)\fP, \fBcorectl\-version(1)\fP
31+
\fBcorectl\-kill(1)\fP, \fBcorectl\-load(1)\fP, \fBcorectl\-ls(1)\fP, \fBcorectl\-panic(1)\fP, \fBcorectl\-ps(1)\fP, \fBcorectl\-pull(1)\fP, \fBcorectl\-put(1)\fP, \fBcorectl\-query(1)\fP, \fBcorectl\-rm(1)\fP, \fBcorectl\-run(1)\fP, \fBcorectl\-ssh(1)\fP, \fBcorectl\-version(1)\fP
3232

3333

3434
.SH HISTORY

‎documentation/man/corectl_panic.1‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
.TH "corectl" "1" "" " " ""
2+
.nh
3+
.ad l
4+
5+
6+
.SH NAME
7+
.PP
8+
corectl\-panic \- Hard kills a running CoreOS instance
9+
10+
11+
.SH SYNOPSIS
12+
.PP
13+
\fBcorectl panic [VMids]\fP
14+
15+
16+
.SH DESCRIPTION
17+
.PP
18+
Hard kills a running CoreOS instance.
19+
This feature is intended as a practical way to test and reproduce both cluster failure scenarios and layouts resilient to them
20+
21+
22+
.SH OPTIONS
23+
.PP
24+
\fB\-r\fP, \fB\-\-random\fP[=false]
25+
hard kill a randomly choosen running instance
26+
27+
28+
.SH OPTIONS INHERITED FROM PARENT COMMANDS
29+
.PP
30+
\fB\-d\fP, \fB\-\-debug\fP[=false]
31+
adds additional verbosity, and options, directed at debugging purposes and power users
32+
33+
34+
.SH SEE ALSO
35+
.PP
36+
\fBcorectl(1)\fP
37+
38+
39+
.SH HISTORY
40+
.PP

‎documentation/markdown/corectl.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ corectl
2323
* [corectl kill](corectl_kill.md) - Halts one or more running CoreOS instances
2424
* [corectl load](corectl_load.md) - Loads CoreOS instances defined in an instrumentation file.
2525
* [corectl ls](corectl_ls.md) - Lists the CoreOS images available locally
26+
* [corectl panic](corectl_panic.md) - Hard kills a running CoreOS instance
2627
* [corectl ps](corectl_ps.md) - Lists running CoreOS instances
2728
* [corectl pull](corectl_pull.md) - Pulls a CoreOS image from upstream
2829
* [corectl put](corectl_put.md) - copy file to inside VM
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
## corectl panic
2+
3+
Hard kills a running CoreOS instance
4+
5+
### Synopsis
6+
7+
8+
Hard kills a running CoreOS instance.
9+
This feature is intended as a practical way to test and reproduce both cluster failure scenarios and layouts resilient to them
10+
11+
```
12+
corectl panic [VMids]
13+
```
14+
15+
### Options
16+
17+
```
18+
-r, --random hard kill a randomly choosen running instance
19+
```
20+
21+
### Options inherited from parent commands
22+
23+
```
24+
-d, --debug adds additional verbosity, and options, directed at debugging purposes and power users
25+
```
26+
27+
### SEE ALSO
28+
* [corectl](corectl.md) - CoreOS over macOS made simple.
29+

0 commit comments

Comments
 (0)
⌂ 换网址