144
197
// Override the arch constraint with the arch of the host.
145
198
return h.f.FindTools(v, series, &version.Current.Arch)
201
// resolvConf is the full path to the resolv.conf file on the local
202
// system. Defined here so it can be overriden for testing.
203
var resolvConf = "/etc/resolv.conf"
205
// localDNSServers parses the /etc/resolv.conf file (if available) and
206
// extracts all nameservers addresses, and the default search domain
208
func localDNSServers() ([]network.Address, string, error) {
209
file, err := os.Open(resolvConf)
210
if os.IsNotExist(err) {
212
} else if err != nil {
213
return nil, "", errors.Annotatef(err, "cannot open %q", resolvConf)
217
var addresses []network.Address
218
var searchDomain string
219
scanner := bufio.NewScanner(file)
221
line := strings.TrimSpace(scanner.Text())
222
if strings.HasPrefix(line, "#") {
226
if strings.HasPrefix(line, "nameserver") {
227
address := strings.TrimPrefix(line, "nameserver")
228
// Drop comments after the address, if any.
229
if strings.Contains(address, "#") {
230
address = address[:strings.Index(address, "#")]
232
address = strings.TrimSpace(address)
233
addresses = append(addresses, network.NewAddress(address))
235
if strings.HasPrefix(line, "search") {
236
searchDomain = strings.TrimPrefix(line, "search")
237
// Drop comments after the domain, if any.
238
if strings.Contains(searchDomain, "#") {
239
searchDomain = searchDomain[:strings.Index(searchDomain, "#")]
241
searchDomain = strings.TrimSpace(searchDomain)
245
if err := scanner.Err(); err != nil {
246
return nil, "", errors.Annotatef(err, "cannot read DNS servers from %q", resolvConf)
248
return addresses, searchDomain, nil
251
// ipRouteAdd is the command template to add a static route for
252
// .ContainerIP using the .HostBridge device (usually lxcbr0 or virbr0).
253
var ipRouteAdd = mustParseTemplate("ipRouteAdd", `
254
ip route add {{.ContainerIP}} dev {{.HostBridge}}`[1:])
256
type IptablesRule struct {
262
var skipSNATRule = IptablesRule{
263
// For EC2, to get internet access we need traffic to appear with
264
// source address matching the container's host. For internal
265
// traffic we want to keep the container IP because it is used
266
// by some services. This rule sits above the SNAT rule, which
267
// changes the source address of traffic to the container host IP
268
// address, skipping this modification if the traffic destination
269
// is inside the EC2 VPC.
272
"-d {{.SubnetCIDR}} -o {{.HostIF}} -j RETURN",
275
var iptablesRules = map[string]IptablesRule{
276
// iptablesCheckSNAT is the command template to verify if a SNAT
277
// rule already exists for the host NIC named .HostIF (usually
278
// eth0) and source address .HostIP (usually eth0's address). We
279
// need to check whether the rule exists because we only want to
280
// add it once. Exit code 0 means the rule exists, 1 means it
285
"-o {{.HostIF}} -j SNAT --to-source {{.HostIP}}",
286
}, "iptablesForwardOut": {
287
// Ensure that we have ACCEPT rules that apply to the containers that
288
// we are creating so any DROP rules added by libvirt while setting
289
// up virbr0 further down the chain don't disrupt wanted traffic.
292
"-d {{.ContainerCIDR}} -o {{.HostBridge}} -j ACCEPT",
293
}, "iptablesForwardIn": {
296
"-s {{.ContainerCIDR}} -i {{.HostBridge}} -j ACCEPT",
299
// mustParseTemplate works like template.Parse, but panics on error.
300
func mustParseTemplate(name, source string) *template.Template {
301
templ, err := template.New(name).Parse(source)
308
// mustExecTemplate works like template.Parse followed by template.Execute,
309
// but panics on error.
310
func mustExecTemplate(name, tmpl string, data interface{}) string {
311
t := mustParseTemplate(name, tmpl)
313
if err := t.Execute(&buf, data); err != nil {
319
// runTemplateCommand executes the given template with the given data,
320
// which generates a command to execute. If exitNonZeroOK is true, no
321
// error is returned if the exit code is not 0, otherwise an error is
323
func runTemplateCommand(t *template.Template, exitNonZeroOK bool, data interface{}) (
324
exitCode int, err error,
326
// Clone the template to ensure the original won't be changed.
327
cloned, err := t.Clone()
329
return -1, errors.Annotatef(err, "cannot clone command template %q", t.Name())
332
if err := cloned.Execute(&buf, data); err != nil {
333
return -1, errors.Annotatef(err, "cannot execute command template %q", t.Name())
335
command := buf.String()
336
logger.Debugf("running command %q", command)
337
result, err := exec.RunCommands(exec.RunParams{Commands: command})
339
return -1, errors.Annotatef(err, "cannot run command %q", command)
341
exitCode = result.Code
342
stdout := string(result.Stdout)
343
stderr := string(result.Stderr)
345
"command %q returned code=%d, stdout=%q, stderr=%q",
346
command, exitCode, stdout, stderr,
352
return exitCode, errors.Errorf(
353
"command %q failed with exit code %d",
360
// setupRoutesAndIPTables sets up on the host machine the needed
361
// iptables rules and static routes for an addressable container.
362
var setupRoutesAndIPTables = func(
364
primaryAddr network.Address,
366
ifaceInfo []network.InterfaceInfo,
370
if primaryNIC == "" || primaryAddr.Value == "" || bridgeName == "" || len(ifaceInfo) == 0 {
371
return errors.Errorf("primaryNIC, primaryAddr, bridgeName, and ifaceInfo must be all set")
374
for _, iface := range ifaceInfo {
375
containerIP := iface.Address.Value
376
if containerIP == "" {
377
return errors.Errorf("container IP %q must be set", containerIP)
386
}{primaryNIC, primaryAddr.Value, bridgeName, containerIP, iface.CIDR, iface.CIDR}
388
var addRuleIfDoesNotExist = func(name string, rule IptablesRule) error {
389
check := mustExecTemplate("rule", "iptables -t {{.Table}} -C {{.Chain}} {{.Rule}}", rule)
390
t := mustParseTemplate(name+"Check", check)
392
code, err := runTemplateCommand(t, true, data)
394
return errors.Trace(err)
398
// Rule does exist. Do nothing
400
// Rule does not exist, add it. We insert the rule at the top of the list so it precedes any
402
action := mustExecTemplate("action", "iptables -t {{.Table}} -I {{.Chain}} 1 {{.Rule}}", rule)
403
t = mustParseTemplate(name+"Add", action)
404
_, err = runTemplateCommand(t, false, data)
406
return errors.Trace(err)
409
// Unexpected code - better report it.
410
return errors.Errorf("iptables failed with unexpected exit code %d", code)
415
for name, rule := range iptablesRules {
416
if !enableNAT && name == "iptablesSNAT" {
417
// Do not add the SNAT rule if we shouldn't enable
421
if err := addRuleIfDoesNotExist(name, rule); err != nil {
426
// TODO(dooferlad): subnets should be a list of subnets in the EC2 VPC and
427
// should be empty for MAAS. See bug http://pad.lv/1443942
429
// Only add the following hack to allow AWS egress traffic
430
// for hosted containers to work.
431
subnets := []string{data.HostIP + "/16"}
432
for _, subnet := range subnets {
433
data.SubnetCIDR = subnet
434
if err := addRuleIfDoesNotExist("skipSNAT", skipSNATRule); err != nil {
440
code, err := runTemplateCommand(ipRouteAdd, false, data)
441
// Ignore errors if the exit code was 2, which signals that the route was not added
442
// because it already exists.
443
if code != 2 && err != nil {
444
return errors.Trace(err)
447
logger.Tracef("route already exists - not added")
449
logger.Tracef("route added: container uses host network interface")
452
logger.Infof("successfully configured iptables and routes for container interfaces")
458
netInterfaces = net.Interfaces
459
interfaceAddrs = (*net.Interface).Addrs
462
// discoverPrimaryNIC returns the name of the first network interface
463
// on the machine which is up and has address, along with the first
465
func discoverPrimaryNIC() (string, network.Address, error) {
466
interfaces, err := netInterfaces()
468
return "", network.Address{}, errors.Annotatef(err, "cannot get network interfaces")
470
logger.Tracef("trying to discover primary network interface")
471
for _, iface := range interfaces {
472
if iface.Flags&net.FlagLoopback != 0 {
473
// Skip the loopback.
474
logger.Tracef("not using loopback interface %q", iface.Name)
477
if iface.Flags&net.FlagUp != 0 {
478
// Possibly the primary, but ensure it has an address as
480
logger.Tracef("verifying interface %q has addresses", iface.Name)
481
addrs, err := interfaceAddrs(&iface)
483
return "", network.Address{}, errors.Annotatef(err, "cannot get %q addresses", iface.Name)
487
// Check if it's an IP or a CIDR.
488
addr := addrs[0].String()
489
ip := net.ParseIP(addr)
492
ip, _, err = net.ParseCIDR(addr)
494
return "", network.Address{}, errors.Annotatef(err, "cannot parse address %q", addr)
499
logger.Tracef("primary network interface is %q, address %q", iface.Name, addr)
500
return iface.Name, network.NewAddress(addr), nil
504
return "", network.Address{}, errors.Errorf("cannot detect the primary network interface")
507
// MACAddressTemplate is used to generate a unique MAC address for a
508
// container. Every 'x' is replaced by a random hexadecimal digit,
509
// while the rest is kept as-is.
510
const MACAddressTemplate = "00:16:3e:xx:xx:xx"
512
// configureContainerNetworking tries to allocate a static IP address
513
// for the given containerId using the provisioner API, when
514
// allocateAddress is true. Otherwise it configures the container with
515
// an already allocated address, when allocateAddress is false (e.g.
516
// after a host reboot). If the API call fails, it's not critical -
517
// just a warning, and it won't cause StartInstance to fail.
518
func configureContainerNetwork(
519
containerId, bridgeDevice string,
521
ifaceInfo []network.InterfaceInfo,
522
allocateAddress bool,
524
) (finalIfaceInfo []network.InterfaceInfo, err error) {
528
"failed configuring a static IP for container %q: %v",
534
if len(ifaceInfo) != 0 {
535
// When we already have interface info, don't overwrite it.
539
var primaryNIC string
540
var primaryAddr network.Address
541
primaryNIC, primaryAddr, err = discoverPrimaryNIC()
543
return nil, errors.Trace(err)
547
logger.Debugf("trying to allocate a static IP for container %q", containerId)
548
finalIfaceInfo, err = apiFacade.PrepareContainerInterfaceInfo(names.NewMachineTag(containerId))
550
logger.Debugf("getting allocated static IP for container %q", containerId)
551
finalIfaceInfo, err = apiFacade.GetContainerInterfaceInfo(names.NewMachineTag(containerId))
554
return nil, errors.Trace(err)
556
logger.Debugf("container interface info result %#v", finalIfaceInfo)
558
// Populate ConfigType and DNSServers as needed.
559
var dnsServers []network.Address
560
var searchDomain string
561
dnsServers, searchDomain, err = localDNSServers()
563
return nil, errors.Trace(err)
565
// Generate the final configuration for each container interface.
566
for i, _ := range finalIfaceInfo {
567
// Always start at the first device index and generate the
568
// interface name based on that. We need to do this otherwise
569
// the container will inherit the host's device index and
571
finalIfaceInfo[i].DeviceIndex = i
572
finalIfaceInfo[i].InterfaceName = fmt.Sprintf("eth%d", i)
573
finalIfaceInfo[i].MACAddress = MACAddressTemplate
574
finalIfaceInfo[i].ConfigType = network.ConfigStatic
575
finalIfaceInfo[i].DNSServers = dnsServers
576
finalIfaceInfo[i].DNSSearch = searchDomain
577
finalIfaceInfo[i].GatewayAddress = primaryAddr
579
err = setupRoutesAndIPTables(
587
return nil, errors.Trace(err)
589
return finalIfaceInfo, nil
592
// MaintainInstance checks that the container's host has the required iptables and routing
593
// rules to make the container visible to both the host and other machines on the same subnet.
594
func (broker *lxcBroker) MaintainInstance(args environs.StartInstanceParams) error {
595
machineId := args.InstanceConfig.MachineId
596
if !environs.AddressAllocationEnabled() {
597
lxcLogger.Debugf("address allocation disabled: Not running maintenance for lxc container with machineId: %s",
602
lxcLogger.Debugf("running maintenance for lxc container with machineId: %s", machineId)
604
// Default to using the host network until we can configure.
605
bridgeDevice := broker.agentConfig.Value(agent.LxcBridge)
606
if bridgeDevice == "" {
607
bridgeDevice = lxc.DefaultLxcBridge
609
_, err := configureContainerNetwork(
614
false, // don't allocate a new address.