Documentation
¶
Index ¶
- Constants
- Variables
- func BackupDirToUse(configuredDir string) string
- func ChecksumSHA256(hexDigest string) string
- func WalkDumps(ctx context.Context, dir string, fn func(name string, reader io.Reader) error) error
- type ArchiveData
- type ArchivePaths
- type ArchiveWorkspace
- type BackupCreator
- type ControllerMetadata
- type CreateArgs
- type DumpEntry
- type DumpExportFunc
- type DumpStaging
- type Manifest
- type ManifestEntry
- type ManifestEntryKind
- type Metadata
- type NamedDump
- type ObjectEntry
- type ObjectSource
- type Origin
- type Paths
Constants ¶
const ( // FilenamePrefix is the prefix used for backup archive files. FilenamePrefix = "juju-backup-" // FilenameTemplate is used with time.Time.Format to generate a filename. FilenameTemplate = FilenamePrefix + "20060102-150405.tar.gz" )
const ChecksumFormatSHA256 = "SHA-256"
ChecksumFormatSHA256 identifies how to interpret the checksum for a backup generated with this version of juju. The encoding is not restated here because the recorded checksum carries it: see ChecksumSHA256.
const UnknownString = "<unknown>"
UnknownString is a marker value for string fields with unknown values.
Variables ¶
var CheckSpaceFor = func(dir string, expectedSize int64) error { total, err := diskTotal(dir) if err != nil { return errors.Capture(err) } diskSizeMargin := float64(total) * 0.10 if diskSizeMargin > minFreeAbsolute { diskSizeMargin = minFreeAbsolute } backupSizeMargin := float64(expectedSize) * 0.20 if backupSizeMargin < diskSizeMargin { backupSizeMargin = diskSizeMargin } wantFree := uint64(expectedSize) + uint64(backupSizeMargin) available, err := diskFree(dir) if err != nil { return errors.Capture(err) } if available < wantFree { return errors.Errorf("not enough free space in %q; want %dMiB, have %dMiB", dir, wantFree/miByte, available/miByte) } return nil }
CheckSpaceFor errors when the free space in dir is less than the expected archive size plus a safety margin. The margin is the larger of the smaller of 5GiB or 10% of the total disk size, and 20% of the expected size. It is a variable so tests can trigger the insufficient-space failure, mirroring Create.
var Create = func(ctx context.Context, meta *Metadata, args CreateArgs) (string, error) { if err := ctx.Err(); err != nil { return "", err } if args.Clock == nil { return "", errors.New("missing clock") } if err := checkDestinationDir(args.DestinationDir); err != nil { return "", errors.Capture(err) } for _, entry := range args.DumpEntries { if err := checkDumpEntryName(entry.Name); err != nil { return "", errors.Capture(err) } } if err := checkObjectEntries(args.ObjectEntries, args.DataDir); err != nil { return "", errors.Capture(err) } stagingDir, err := os.MkdirTemp(args.DestinationDir, tempPrefix) if err != nil { return "", errors.Errorf("making backups staging directory: %w", err) } defer func() { _ = os.RemoveAll(stagingDir) }() archivePaths := NewNonCanonicalArchivePaths(stagingDir) if err := os.MkdirAll(archivePaths.DBDumpDir, 0700); err != nil { return "", errors.Errorf("creating temp directories: %w", err) } meta.Finished = new(args.Clock.Now()) metadataReader, err := meta.AsJSONBuffer() if err != nil { return "", errors.Errorf("preparing the metadata: %w", err) } canonicalPaths := NewCanonicalArchivePaths() metaSize, metaSum, err := writeAll(archivePaths.MetadataFile, metadataReader) if err != nil { return "", errors.Capture(err) } manifestEntries := []ManifestEntry{{ Path: canonicalPaths.MetadataFile, Kind: ManifestKindMetadata, Size: metaSize, SHA256: metaSum, }} filesSize, filesSum, err := buildFilesArchive(ctx, archivePaths.FilesArchive, args.FilesToBackUp, args.DataDir, args.ObjectEntries) if err != nil { return "", errors.Capture(err) } manifestEntries = append(manifestEntries, ManifestEntry{ Path: canonicalPaths.FilesArchive, Kind: ManifestKindFilesArchive, Size: filesSize, SHA256: filesSum, }) entries := make([]DumpEntry, len(args.DumpEntries)) for i, entry := range args.DumpEntries { entries[i] = DumpEntry{Name: entry.Name, Reader: contextReader{ctx: ctx, Reader: entry.Reader}} } dumpEntries, err := buildDump( archivePaths.DBDumpDir, canonicalPaths.DBDumpDir, entries) if err != nil { return "", errors.Capture(err) } manifestEntries = append(manifestEntries, dumpEntries...) manifestReader, err := NewManifest(manifestEntries).AsJSONBuffer() if err != nil { return "", errors.Errorf("preparing the manifest: %w", err) } if _, _, err := writeAll(archivePaths.ManifestFile, manifestReader); err != nil { return "", errors.Capture(err) } filename := filepath.Join(args.DestinationDir, meta.Started.Format(FilenameTemplate)) size, checksum, err := buildArchiveAndChecksum(ctx, filename, stagingDir, archivePaths.ContentDir) if err != nil { return "", errors.Capture(err) } if err := meta.MarkComplete(size, checksum, args.Clock.Now()); err != nil { return "", discardArchive(filename, errors.Errorf("updating metadata: %w", err)) } if err := ctx.Err(); err != nil { return "", discardArchive(filename, err) } return filename, nil }
Create builds a new backup archive file in args.DestinationDir, named after meta.Started using FilenameTemplate. It updates the metadata with the file info and returns the archive filename. It is a variable so tests can replace the archive creation with a stub, mirroring GetFilesToBackUp.
var GetFilesToBackUp = func(rootDir string, paths *Paths) ([]string, error) { glob := filepath.Join(rootDir, paths.DataDir, agentsDir, agentsConfs) agentConfs, err := filepath.Glob(glob) if err != nil { return nil, errors.Errorf("failed to fetch agent config files: %w", err) } glob = filepath.Join(rootDir, paths.DataDir, initDir, "*") serviceConfs, err := filepath.Glob(glob) if err != nil { return nil, errors.Errorf("failed to fetch service config files: %w", err) } backupFiles := []string{ filepath.Join(rootDir, paths.DataDir, toolsDir), } backupFiles = append(backupFiles, agentConfs...) backupFiles = append(backupFiles, serviceConfs...) sshAuthKeysPath := filepath.Join(rootDir, SSHDir, authKeysFile) optional := []string{ filepath.Join(rootDir, paths.DataDir, sshIdentFile), filepath.Join(rootDir, paths.DataDir, serverPEM), filepath.Join(rootDir, paths.DataDir, dbSecret), filepath.Join(rootDir, paths.DataDir, nonceFile), sshAuthKeysPath, } for _, file := range optional { if _, err := os.Stat(file); err != nil { if errors.Is(err, os.ErrNotExist) { continue } if errors.Is(err, os.ErrPermission) && file == sshAuthKeysPath { continue } return nil, errors.Capture(err) } backupFiles = append(backupFiles, file) } // The object store directory is intentionally not collected here: // object contents reach the archive through the object entries // derived from the database dumps, which works for both the file // and S3 backends, so a backup does not depend on a local cache of // S3 objects. var finalBackupFiles []string for _, file := range backupFiles { err := filepath.Walk(file, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { return nil } if info.Mode().IsRegular() { finalBackupFiles = append(finalBackupFiles, path) } else if info.Mode()&os.ModeSymlink != 0 { if _, err := os.Stat(path); err != nil { if errors.Is(err, os.ErrNotExist) { return nil } return err } finalBackupFiles = append(finalBackupFiles, path) } return nil }) if err != nil { return nil, errors.Errorf("cannot walk %q: %w", file, err) } } return finalBackupFiles, nil }
GetFilesToBackUp returns the paths that should be included in the backup archive. It is a variable so tests can replace the filesystem walk with a deterministic list, mirroring SSHDir.
var SSHDir = "/home/ubuntu/.ssh"
SSHDir is the host directory holding the ubuntu user's SSH files. It is a variable so tests can redirect the lookup into their own tree: on hosts where the directory exists but is not readable by the test user, the fatal stat failure would make results depend on the test runner.
var StageDumps = func(ctx context.Context, dir string, dumps []NamedDump) (*DumpStaging, error) { stagingDir, err := os.MkdirTemp(dir, "juju-backup-dump-") if err != nil { return nil, errors.Capture(err) } staging := &DumpStaging{dir: stagingDir} defer func() { if staging != nil { _ = staging.Close() } }() for _, dump := range dumps { file, err := stageDump(ctx, stagingDir, dump.Name, dump.Export) if err != nil { return nil, err } staging.files = append(staging.files, file) staging.entries = append(staging.entries, DumpEntry{ Name: dump.Name, Reader: file, }) } result := staging staging = nil return result, nil }
StageDumps marshals each dump into a temporary directory inside dir and returns the staging handle holding the resulting entries. The caller must create dir beforehand and must keep the handle open until the dumps have been archived, then Close it. It is a variable so tests can wrap the staging, mirroring Create.
var UnknownInt64 = int64(math.MaxInt64)
UnknownInt64 is a marker value for int64 fields with unknown values.
var UnknownVersion = semversion.MustParse("9999.9999.9999")
UnknownVersion is a marker value for version fields with unknown values.
Functions ¶
func BackupDirToUse ¶
BackupDirToUse returns the desired backup staging dir.
func ChecksumSHA256 ¶
ChecksumSHA256 returns the canonical recorded form of a hex SHA-256 digest: the algorithm prefix followed by the digest. A digest already carrying the prefix is returned unchanged.
func WalkDumps ¶
WalkDumps reads every staged dump back from dir, passing each file's archive-relative name and contents to fn. It is the read side of StageDumps, used when recovering a backup.
The fn callback receives each file as a reader that will be closed when fn returns. The callback must fully consume the reader before returning; any data left unread is lost because the underlying file is closed immediately after fn returns. Storing the reader for later use (e.g. in a goroutine) is not safe.
Types ¶
type ArchiveData ¶
type ArchiveData struct {
ArchivePaths
// contains filtered or unexported fields
}
ArchiveData is a wrapper around a the uncompressed data in a backup archive file. It provides access to the content of the archive. While ArchiveData provides useful functionality, it may not be appropriate for large archives. The contents of the archive are kept in-memory, so large archives could be too taxing on the host. In that case consider using ArchiveWorkspace instead.
func NewArchiveData ¶
func NewArchiveData(data []byte) *ArchiveData
NewArchiveData builds a new archive data wrapper for the given uncompressed data.
func NewArchiveDataReader ¶
func NewArchiveDataReader(r io.Reader) (*ArchiveData, error)
NewArchiveDataReader returns a new archive data wrapper for the data in the provided reader. Note that the entire archive will be read into memory and kept there. So for relatively large archives it will often be more appropriate to use ArchiveWorkspace instead.
func (*ArchiveData) Metadata ¶
func (ad *ArchiveData) Metadata() (*Metadata, error)
Metadata returns the metadata stored in the backup archive. If no metadata is there, errors.NotFound is returned.
func (*ArchiveData) NewBuffer ¶
func (ad *ArchiveData) NewBuffer() *bytes.Buffer
NewBuffer wraps the archive data in a Buffer.
func (*ArchiveData) Version ¶
func (ad *ArchiveData) Version() (*semversion.Number, error)
Version returns the juju version under which the backup archive was created. If no version is found in the archive, it must come from before backup archives included the version. In that case we return version 1.20.
type ArchivePaths ¶
type ArchivePaths struct {
// ContentDir is the path to the directory within the archive
// containing all the contents. It is the only file or directory at
// the top-level of the archive and everything else in the archive
// is contained in the content directory.
ContentDir string
// FilesArchive is the path to the tar file inside the archive
// containing all the database related files (with the exception of the
// DB dump files) gathered in by the backup machinery.
FilesArchive string
// DBDumpDir is the path to the directory within the archive
// contents that contains all the files dumped from the juju
// database.
DBDumpDir string
// MetadataFile is the path to the metadata file.
MetadataFile string
// ManifestFile is the path to the content manifest: the index of
// the archive's components with their sizes and SHA-256 hashes.
ManifestFile string
}
ArchivePaths holds the paths to the files and directories in a backup archive.
func NewCanonicalArchivePaths ¶
func NewCanonicalArchivePaths() ArchivePaths
NewCanonicalArchivePaths composes a new ArchivePaths with default values set. These values are relative (un-rooted) and the canonical slash ("/") is the path separator. Thus the paths are suitable for resolving the paths in a backup archive file (which is a tar file).
func NewNonCanonicalArchivePaths ¶
func NewNonCanonicalArchivePaths(rootDir string) ArchivePaths
NewNonCanonicalArchivePaths builds a new ArchivePaths using default values, rooted at the provided rootDir. The path separator used is platform-dependent. The resulting paths are suitable for locating backup archive contents in a directory into which an archive has been unpacked.
type ArchiveWorkspace ¶
type ArchiveWorkspace struct {
ArchivePaths
RootDir string
}
ArchiveWorkspace is a wrapper around backup archive info that has a concrete root directory and an archive unpacked in it.
func NewArchiveWorkspaceReader ¶
func NewArchiveWorkspaceReader(archive io.Reader) (*ArchiveWorkspace, error)
NewArchiveWorkspaceReader returns a new archive workspace with a new workspace dir populated from the archive. Note that this involves unpacking the entire archive into a directory under the host's "temporary" directory. For relatively large archives this could have adverse effects on hosts with little disk space.
func (*ArchiveWorkspace) Close ¶
func (ws *ArchiveWorkspace) Close() error
Close cleans up the workspace dir.
func (*ArchiveWorkspace) Metadata ¶
func (ws *ArchiveWorkspace) Metadata() (*Metadata, error)
Metadata returns the metadata derived from the JSON file in the archive.
func (*ArchiveWorkspace) OpenArchivedFile ¶
func (ws *ArchiveWorkspace) OpenArchivedFile(filename string) (io.Reader, error)
OpenArchivedFile returns an open ReadCloser for the corresponding file in root.tar.
func (*ArchiveWorkspace) UnpackFilesArchive ¶
func (ws *ArchiveWorkspace) UnpackFilesArchive(targetRoot string) error
UnpackFilesArchive extracts root.tar into the target directory.
type BackupCreator ¶
type BackupCreator interface {
// Create creates an archive with the supplied notes. It honours context
// cancellation and returns only after generation and lease maintenance stop.
// On success, the caller must invoke cleanup after consuming the archive.
// On error, the implementation removes any partial or completed archive.
Create(ctx context.Context, notes string) (meta *Metadata, archive string, cleanup func(), err error)
}
type ControllerMetadata ¶
type ControllerMetadata struct {
// UUID contains the controller UUID.
UUID string
// MachineID contains controller machine id from which this backup is taken.
MachineID string
// MachineInstanceID contains controller machine's instance id from which this backup is taken.
MachineInstanceID string
// HANodes contains the number of nodes in this controller's HA configuration.
HANodes int64
}
ControllerMetadata contains controller specific metadata.
func UnknownController ¶
func UnknownController() ControllerMetadata
UnknownController returns a new backups origin with unknown values.
type CreateArgs ¶
type CreateArgs struct {
// DestinationDir is the absolute path to the directory in which
// the archive is stored. The staging area is created there too.
DestinationDir string
// DataDir is the absolute path to the source controller's data
// directory; object entries are stored relative to it. Required
// when ObjectEntries is not empty.
DataDir string
// FilesToBackUp is the list of absolute paths to the files to
// include in the archive's root.tar.
FilesToBackUp []string
// ObjectEntries are the object store objects archived into the
// archive's root.tar, streamed and validated one at a time.
ObjectEntries []ObjectEntry
// DumpEntries are the database dumps written under the archive's
// dump directory.
DumpEntries []DumpEntry
// Clock provides the time stamping the metadata when the archive
// creation finishes. It must not be nil.
Clock clock.Clock
}
CreateArgs holds the arguments for building a backup archive.
type DumpEntry ¶
type DumpEntry struct {
// Name is the path of the entry relative to the archive's dump
// directory, e.g. "controller.yaml" or "models/<uuid>.yaml".
Name string
// Reader provides the contents of the dump entry.
Reader io.Reader
}
DumpEntry is a single database dump file to include in the backup archive, under the archive's dump directory.
type DumpExportFunc ¶
DumpExportFunc marshals one database dump into writer. Implementations must stream (e.g. yaml.Encoder), so the dump's bytes never accumulate in memory.
func YAMLDump ¶
func YAMLDump(v any) DumpExportFunc
YAMLDump returns a DumpExportFunc that streams v as YAML into the writer.
type DumpStaging ¶
type DumpStaging struct {
// contains filtered or unexported fields
}
DumpStaging is a directory of staged database dumps. Its files feed directly into Create as DumpEntry readers. Close must be called once the dumps have been archived; it releases the readers and removes the staging directory, so a failed archive creation leaves no partial dumps behind.
func (*DumpStaging) Close ¶
func (s *DumpStaging) Close() error
Close closes the staged dump readers and removes the staging directory. Close and remove failures are returned rather than swallowed, but the caller is expected to invoke Close from a deferred cleanup path and only log the error: it must not mask the result of the operation Close is cleaning up after.
func (*DumpStaging) Entries ¶
func (s *DumpStaging) Entries() []DumpEntry
Entries returns the dump entries to pass to Create.
func (*DumpStaging) EntryExceedsLimit ¶
func (s *DumpStaging) EntryExceedsLimit(limit int64) (name string, size int64, err error)
EntryExceedsLimit reports the staged dump exceeding limit, if any. Recovery refuses dumps larger than its own read bound, so a dump that would back up fine but can never be recovered fails the backup instead. The entry name is empty when nothing exceeds the limit.
func (*DumpStaging) Size ¶
func (s *DumpStaging) Size() (int64, error)
Size returns the total size in bytes of the staged dumps.
type Manifest ¶
type Manifest struct {
FormatVersion int64 `json:"format-version"`
Files []ManifestEntry `json:"files"`
}
Manifest is the content index of a backup archive. Unlike metadata.json — the archive's provenance record — the manifest inventories the archive's components with their sizes and content hashes, so a reader can verify each component independently and localise corruption without trusting the archive as a whole.
The manifest lives inside the archive: it inherits the archive's trust level and can never carry the outer archive checksum, which remains the operator-supplied trust anchor.
func NewManifest ¶
func NewManifest(entries []ManifestEntry) *Manifest
NewManifest builds the manifest for the given entries.
func NewManifestJSONReader ¶
NewManifestJSONReader extracts a manifest from its JSON file. Only the current format version is decoded: a newer manifest is rejected rather than silently read with the wrong semantics.
type ManifestEntry ¶
type ManifestEntry struct {
Path string `json:"path"`
Kind ManifestEntryKind `json:"kind"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
ModelUUID string `json:"model-uuid,omitempty"`
}
ManifestEntry records one archive component: its path inside the archive, what it is, and its uncompressed size and SHA-256 content hash, hex encoded.
type ManifestEntryKind ¶
type ManifestEntryKind string
ManifestEntryKind classifies the archive component a manifest entry records.
const ( // ManifestKindMetadata is the archive's metadata.json. ManifestKindMetadata ManifestEntryKind = "metadata" // ManifestKindFilesArchive is the root.tar archive of data-directory // files. ManifestKindFilesArchive ManifestEntryKind = "files-archive" // ManifestKindControllerDump is the controller database dump. ManifestKindControllerDump ManifestEntryKind = "controller-dump" // ManifestKindModelDump is a model database dump. ManifestKindModelDump ManifestEntryKind = "model-dump" )
func ClassifyManifestPath ¶
func ClassifyManifestPath(archivePath string) (ManifestEntryKind, string, error)
ClassifyManifestPath returns the manifest kind for a canonical archive path, and the model UUID for model dumps. Any path outside the known archive components is an error: the archive carries nothing the writer cannot account for.
type Metadata ¶
type Metadata struct {
*filestorage.FileMetadata
// Started records when the backup was started.
Started time.Time
// Finished records when the backup was complete.
Finished *time.Time
// Origin identifies where the backup was created.
Origin Origin
// Notes is an optional user-supplied annotation.
Notes string
// FormatVersion stores format version of these metadata.
FormatVersion int64
// Controller contains metadata about the controller where the backup was taken.
Controller ControllerMetadata
}
Metadata contains the metadata for a single backup archive.
func BuildMetadata ¶
BuildMetadata generates the metadata for a backup archive file.
func NewMetadata ¶
NewMetadata returns a new Metadata for a backup archive, in the most current format, started at the given time.
func NewMetadataJSONReader ¶
NewMetadataJSONReader extracts a new metadata from the JSON file.
func (*Metadata) AsJSONBuffer ¶
AsJSONBuffer returns a bytes.Buffer containing the JSON-ified metadata. This will always produce latest known format.
type NamedDump ¶
type NamedDump struct {
Name string
Export DumpExportFunc
}
NamedDump pairs a dump entry name with the function producing its contents, e.g. "controller.yaml" or "models/<uuid>.yaml".
type ObjectEntry ¶
type ObjectEntry struct {
// Namespace is the object store namespace the object belongs to:
// "controller" for the controller database, or a model UUID.
Namespace string
// SHA256 is the hex-encoded SHA-256 of the object, as exported by
// the database. It identifies the object for reading.
SHA256 string
// SHA384 is the hex-encoded SHA-384 of the object, as exported by
// the database. It names the object inside the archive.
SHA384 string
// Size is the expected object size in bytes.
Size int64
// Source opens the object for reading.
Source ObjectSource
}
ObjectEntry is a single object store object to include in the backup's root.tar archive.
type ObjectSource ¶
type ObjectSource func(ctx context.Context) (io.ReadCloser, error)
ObjectSource opens the content of one object store object for reading. The returned reader must be closed by the caller.
type Origin ¶
type Origin struct {
Model string
Machine string
Hostname string
Version semversion.Number
Base string
}
Origin identifies where a backup archive came from. While it is more about where and Metadata about what and when, that distinction does not merit special consideration. Instead, Origin exists separately from Metadata due to its use as an argument when requesting the creation of a new backup.
func UnknownOrigin ¶
func UnknownOrigin() Origin
UnknownOrigin returns a new backups origin with unknown values.