Merge pull request #60 from kreczko/towards_2.0

Merging development branch into master

.gitignore

@@ -1 +1,3 @@
 files/pool_password
+Gemfile.lock
+pkg

.travis.yml

@@ -1,17 +1,18 @@
 ---
+sudo: false
 language: ruby
+cache: bundler
 bundler_args: --without development
 script:
-  - puppet --version 
   - "bundle exec rake validate && bundle exec rake lint && bundle exec rake spec SPEC_OPTS='--format documentation'"
 matrix:
   fast_finish: true
   include:
-  - rvm: 1.9.3
+  - rvm: 2.1.7
     env: PUPPET_GEM_VERSION="~> 3.0"
-  - rvm: 2.1.5
-    env: PUPPET_GEM_VERSION="~> 3.0"
-  - rvm: 2.1.6
-    env: PUPPET_GEM_VERSION="~> 4.0" STRICT_VARIABLES="yes"
+  - rvm: 2.1.7
+    env: PUPPET_GEM_VERSION="~> 4.0"
+  - rvm: 2.3.1
+    env: PUPPET_GEM_VERSION="~> 4.0"
 notifications:
   email: false

CHANGELOG.md

@@ -0,0 +1,132 @@
+# Change Log
+
+## [v1.3.1](https://github.com/hep-puppet/htcondor/tree/v1.3.1) (2017-05-18)
+[Full Changelog](https://github.com/hep-puppet/htcondor/compare/v1.3.0...v1.3.1)
+
+**Implemented enhancements:**
+
+- Add profiles [\#46](https://github.com/HEP-Puppet/htcondor/issues/46)
+- Simplify parameters [\#44](https://github.com/HEP-Puppet/htcondor/issues/44)
+- Repository clean up [\#43](https://github.com/HEP-Puppet/htcondor/issues/43)
+- Towards version 2.0 - part 3 [\#55](https://github.com/HEP-Puppet/htcondor/pull/55) ([kreczko](https://github.com/kreczko))
+- Step 2 towards version 2.0 [\#54](https://github.com/HEP-Puppet/htcondor/pull/54) ([kreczko](https://github.com/kreczko))
+- Big simplifications [\#53](https://github.com/HEP-Puppet/htcondor/pull/53) ([kreczko](https://github.com/kreczko))
+
+**Closed issues:**
+
+- security considerations ? [\#42](https://github.com/HEP-Puppet/htcondor/issues/42)
+- wrong permissions if condor user != root [\#38](https://github.com/HEP-Puppet/htcondor/issues/38)
+
+**Merged pull requests:**
+
+- Version 1.3.1 [\#52](https://github.com/HEP-Puppet/htcondor/pull/52) ([kreczko](https://github.com/kreczko))
+- \[New feature\] high-availability deployment for multiple managers [\#51](https://github.com/HEP-Puppet/htcondor/pull/51) ([kreczko](https://github.com/kreczko))
+- 2016 spring clean [\#50](https://github.com/HEP-Puppet/htcondor/pull/50) ([kreczko](https://github.com/kreczko))
+
+## [v1.3.0](https://github.com/hep-puppet/htcondor/tree/v1.3.0) (2016-01-29)
+[Full Changelog](https://github.com/hep-puppet/htcondor/compare/v1.2.0...v1.3.0)
+
+**Implemented enhancements:**
+
+- bumping version to 1.2.0 + tidying up [\#40](https://github.com/HEP-Puppet/htcondor/pull/40) ([kreczko](https://github.com/kreczko))
+
+**Closed issues:**
+
+- Publish on Puppet Forge [\#39](https://github.com/HEP-Puppet/htcondor/issues/39)
+- Swap repositories for dependency on grid\_repos [\#3](https://github.com/HEP-Puppet/htcondor/issues/3)
+
+**Merged pull requests:**
+
+- Dropped Puppet 2.7 from tests and added Puppet 4.0 [\#47](https://github.com/HEP-Puppet/htcondor/pull/47) ([kreczko](https://github.com/kreczko))
+- enable cgroup [\#41](https://github.com/HEP-Puppet/htcondor/pull/41) ([kashif74](https://github.com/kashif74))
+
+## [v1.2.0](https://github.com/hep-puppet/htcondor/tree/v1.2.0) (2014-11-12)
+[Full Changelog](https://github.com/hep-puppet/htcondor/compare/v1.1.0...v1.2.0)
+
+**Merged pull requests:**
+
+- Added kerberos map file [\#37](https://github.com/HEP-Puppet/htcondor/pull/37) ([kreczko](https://github.com/kreczko))
+- setting default FILESYSTEM\_DOMAIN to FQDN \(not all WNs have shared FS\) [\#36](https://github.com/HEP-Puppet/htcondor/pull/36) ([kreczko](https://github.com/kreczko))
+- Improving auth method setting [\#35](https://github.com/HEP-Puppet/htcondor/pull/35) ([kreczko](https://github.com/kreczko))
+- Condor id fix, better tests and puppet-lint fixes [\#33](https://github.com/HEP-Puppet/htcondor/pull/33) ([kreczko](https://github.com/kreczko))
+
+## [v1.1.0](https://github.com/hep-puppet/htcondor/tree/v1.1.0) (2014-10-16)
+[Full Changelog](https://github.com/hep-puppet/htcondor/compare/v1.0.0...v1.1.0)
+
+**Closed issues:**
+
+- htcondor files ownership [\#14](https://github.com/HEP-Puppet/htcondor/issues/14)
+
+**Merged pull requests:**
+
+- Exclude condor-i686 and few other bits [\#32](https://github.com/HEP-Puppet/htcondor/pull/32) ([kashif74](https://github.com/kashif74))
+- A few small changes [\#31](https://github.com/HEP-Puppet/htcondor/pull/31) ([kreczko](https://github.com/kreczko))
+- New feature: ganglia [\#30](https://github.com/HEP-Puppet/htcondor/pull/30) ([kreczko](https://github.com/kreczko))
+- New feature: kerberos [\#29](https://github.com/HEP-Puppet/htcondor/pull/29) ([kreczko](https://github.com/kreczko))
+- Queues with hiera or config [\#28](https://github.com/HEP-Puppet/htcondor/pull/28) ([kreczko](https://github.com/kreczko))
+- Fairshares updated [\#27](https://github.com/HEP-Puppet/htcondor/pull/27) ([kreczko](https://github.com/kreczko))
+- email, default and filesystem domains [\#26](https://github.com/HEP-Puppet/htcondor/pull/26) ([kreczko](https://github.com/kreczko))
+- Request memory [\#25](https://github.com/HEP-Puppet/htcondor/pull/25) ([kreczko](https://github.com/kreczko))
+- adding option for DEV repositories [\#24](https://github.com/HEP-Puppet/htcondor/pull/24) ([kreczko](https://github.com/kreczko))
+- Defrag and partitionable slots [\#23](https://github.com/HEP-Puppet/htcondor/pull/23) ([kreczko](https://github.com/kreczko))
+- fix ":" -\> "=" [\#22](https://github.com/HEP-Puppet/htcondor/pull/22) ([kreczko](https://github.com/kreczko))
+
+## [v1.0.0](https://github.com/hep-puppet/htcondor/tree/v1.0.0) (2014-08-07)
+[Full Changelog](https://github.com/hep-puppet/htcondor/compare/New features...v1.0.0)
+
+**Closed issues:**
+
+- Generic template for fair shares [\#8](https://github.com/HEP-Puppet/htcondor/issues/8)
+- Fix fair shares and groups [\#4](https://github.com/HEP-Puppet/htcondor/issues/4)
+
+**Merged pull requests:**
+
+- Added defrag and healthcheck [\#19](https://github.com/HEP-Puppet/htcondor/pull/19) ([kashif74](https://github.com/kashif74))
+- make sure condor\_reconfig is not run before service is up [\#18](https://github.com/HEP-Puppet/htcondor/pull/18) ([fschaer](https://github.com/fschaer))
+- allow user-defined templates to be specified [\#17](https://github.com/HEP-Puppet/htcondor/pull/17) ([fschaer](https://github.com/fschaer))
+- Fix3 [\#16](https://github.com/HEP-Puppet/htcondor/pull/16) ([fschaer](https://github.com/fschaer))
+- specify file ownership and allow for user \(root\) override, as this is [\#15](https://github.com/HEP-Puppet/htcondor/pull/15) ([fschaer](https://github.com/fschaer))
+- be librarian-puppet friendly [\#13](https://github.com/HEP-Puppet/htcondor/pull/13) ([fschaer](https://github.com/fschaer))
+- Changes for seperate scheduler configuartion [\#12](https://github.com/HEP-Puppet/htcondor/pull/12) ([kashif74](https://github.com/kashif74))
+- Fixes for Nagios tests [\#11](https://github.com/HEP-Puppet/htcondor/pull/11) ([kreczko](https://github.com/kreczko))
+- Fairshare fixes [\#10](https://github.com/HEP-Puppet/htcondor/pull/10) ([kreczko](https://github.com/kreczko))
+- Fixes for issues \#4 and \#8 + other stuff [\#9](https://github.com/HEP-Puppet/htcondor/pull/9) ([kreczko](https://github.com/kreczko))
+- Updating things for productin [\#7](https://github.com/HEP-Puppet/htcondor/pull/7) ([kreczko](https://github.com/kreczko))
+- new version [\#6](https://github.com/HEP-Puppet/htcondor/pull/6) ([kashif74](https://github.com/kashif74))
+- Added priority to repo [\#5](https://github.com/HEP-Puppet/htcondor/pull/5) ([kashif74](https://github.com/kashif74))
+- First working version of Puppet module for HTCondor [\#1](https://github.com/HEP-Puppet/htcondor/pull/1) ([kreczko](https://github.com/kreczko))
+
+# Version 2.0.0
+Version 2.0.0 brought big changes to the module. The biggest change is a structural one.
+`htcondor::params.pp` was added to set defaults for all the parameters.
+In addition, parameters are attempted to be read via `hiera` first. Full merge
+support for hashes and arrays is provided.
+With these changes the `htcondor::config.pp` was split into six pieces:
+ - the main config setting up the rest
+ - a common config part
+ - the security configuration
+ - separate configs for manager, scheduler & worker
+The full detail of these changes can be seen in [PR 53](https://github.com/HEP-Puppet/htcondor/pull/53).
+
+## New features
+- configure connection broker for private workers (i.e. workers that cannot be reached from the manager or scheduler but can reach the manager).
+- enabled `ganglia` daemon for schedulers (previously only possible on managers)
+- flag to enable [condor reporting](http://research.cs.wisc.edu/htcondor/privacy.html), disabed by default
+- added `use_anonymous_auth`
+- added `custom_machine_attributes` and `custom_machine_attributes` which can be used to add classads for `STARTD_ATTRS` and `STARTD_JOB_ATTRS`
+
+## Bug fixes
+- daemon list would be incorrect for some versions of Ruby. This was due to the use of `and` and `or` operators which is incorrect for boolean comparisons.
+- added missing `cluster_has_multiple_domains` parameter (w.r.t to 2.0.0 beta)
+- removed repository dependency if it is disabled
+
+## Other
+- changed config templates to ensure new line at the end of the file and reduced the use of `-%>`
+- workers are no longer able to write to schedulers by default
+- new formatting for the security config: one line per entry for manager/scheduler/worker
+- removed `use_pkg_config` parameter.
+- no longer changing `/etc/condor/condor_config` nor `/etc/condor/condor_config.local` as recommended by the HTCondor team
+- content previously in `/etc/condor/condor_config.local` now in `/etc/condor/config.d/00_config_local.config`
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

Gemfile

@@ -1,6 +1,26 @@
 source 'https://rubygems.org'
 
-puppetversion = ENV.key?('PUPPET_VERSION') ? "= #{ENV['PUPPET_VERSION']}" : ['>= 2.7']
-gem 'puppet', puppetversion
-gem 'puppet-lint', '>= 0.3.2'
-gem 'puppetlabs_spec_helper', '>= 0.1.0'
+# Find a location or specific version for a gem. place_or_version can be a
+# version, which is most often used. It can also be git, which is specified as
+# `git://somewhere.git#branch`. You can also use a file source location, which
+# is specified as `file://some/location/on/disk`.
+def location_for(place_or_version, fake_version = nil)
+  if place_or_version =~ /^(git[:@][^#]*)#(.*)/
+    [fake_version, { :git => $1, :branch => $2, :require => false }].compact
+  elsif place_or_version =~ /^file:\/\/(.*)/
+    ['>= 0', { :path => File.expand_path($1), :require => false }]
+  else
+    [place_or_version, { :require => false }]
+  end
+end
+
+supports_windows = false
+
+gem 'puppet', *location_for(ENV['PUPPET_GEM_VERSION'])
+gem 'puppet-lint', '~> 2.0'
+gem 'puppetlabs_spec_helper', '~> 2.1'
+gem 'github_changelog_generator', '<= 1.14.3'
+gem 'rspec-puppet', '~> 2.5'
+gem 'rspec-puppet-facts'
+gem 'rspec-puppet-utils'
+gem 'metadata-json-lint'

HISTORY.md

@@ -0,0 +1,31 @@
+# Version 2.0.0
+Version 2.0.0 brought big changes to the module. The biggest change is a structural one.
+`htcondor::params.pp` was added to set defaults for all the parameters.
+In addition, parameters are attempted to be read via `hiera` first. Full merge
+support for hashes and arrays is provided.
+With these changes the `htcondor::config.pp` was split into six pieces:
+ - the main config setting up the rest
+ - a common config part
+ - the security configuration
+ - separate configs for manager, scheduler & worker
+The full detail of these changes can be seen in [PR 53](https://github.com/HEP-Puppet/htcondor/pull/53).
+
+## New features
+- configure connection broker for private workers (i.e. workers that cannot be reached from the manager or scheduler but can reach the manager).
+- enabled `ganglia` daemon for schedulers (previously only possible on managers)
+- flag to enable [condor reporting](http://research.cs.wisc.edu/htcondor/privacy.html), disabed by default
+- added `use_anonymous_auth`
+- added `custom_machine_attributes` and `custom_machine_attributes` which can be used to add classads for `STARTD_ATTRS` and `STARTD_JOB_ATTRS`
+
+## Bug fixes
+- daemon list would be incorrect for some versions of Ruby. This was due to the use of `and` and `or` operators which is incorrect for boolean comparisons.
+- added missing `cluster_has_multiple_domains` parameter (w.r.t to 2.0.0 beta)
+- removed repository dependency if it is disabled
+
+## Other
+- changed config templates to ensure new line at the end of the file and reduced the use of `-%>`
+- workers are no longer able to write to schedulers by default
+- new formatting for the security config: one line per entry for manager/scheduler/worker
+- removed `use_pkg_config` parameter.
+- no longer changing `/etc/condor/condor_config` nor `/etc/condor/condor_config.local` as recommended by the HTCondor team
+- content previously in `/etc/condor/condor_config.local` now in `/etc/condor/config.d/00_config_local.config`

README.md

@@ -1,20 +1,93 @@
-Puppet module for HTCondor batch system
-=========================================
-Latest stable version: https://github.com/HEP-Puppet/htcondor/releases/tag/v1.3.1
+#Puppet module for HTCondor batch system
 
-Development branch (heading for 2.0.0): https://github.com/HEP-Puppet/htcondor/tree/development
+Latest stable version: https://github.com/HEP-Puppet/htcondor/releases/tag/v2.0.1
+
+Development branch: https://github.com/HEP-Puppet/htcondor/tree/development
 
 [![Build Status](https://travis-ci.org/HEP-Puppet/htcondor.svg?branch=master)](https://travis-ci.org/HEP-Puppet/htcondor)
 
 Puppetforge: https://forge.puppetlabs.com/HEPPuppet/htcondor
 
-Installation requirements
-=========================================
-- pool_password file created with ```condor_store_cred -f <path_to_htcondor_module>/files/pool_password```
 
-Tests
-=========================================
+####Table of Contents
+1. [Overview - What is the htcondor module?](#overview)
+2. [Module Description - What does the module do?](#module-description)
+3. [Setup - The basics of getting started with htcondor](#setup)
+4. [Limitations - OS compatibility, etc.](#limitations)
+7. [Development - Guide for contributing to the module](#development)
+	* [Contributing to the htcondor module](#contributing)
+    * [Running tests - A quick guide](#running-tests)
+
+##Overview
+The htcondor modules allows you to set up a HTCondor cluster (https://research.cs.wisc.edu/htcondor/).
+It depends on several other modules, including puppetlabs/(stdlib|concat|firewall).
+Please check the metadata.json for detailed dependencies.
+
+##Module Description
+An HTCondor cluster consists of at least three types of nodes:
+ * a worker for executing the jobs
+ * a scheduler for job submission
+ * a collector/negotiator to match jobs with workers
+
+This puppet modules allows for the configuration of these three types of nodes.
+
+
+##Setup
+**What the htcondor module affects:**
+ * configuration files and directories (/etc/condor/*)
+ * installation of htcondor software (condor* packages)
+ * a new fact for facter: condor_version
+
+###Beginning with HTCondor
+Since admins might wish to run their own repository or disable repositories after install,
+the HTCondor repository is no longer included in the Puppet module since version 2.0.0.
+Therefore, the first step is to install the latest HTCondor repository for your OS (https://research.cs.wisc.edu/htcondor/yum/):
+```
+yum install -y https://research.cs.wisc.edu/htcondor/yum/repo.d/htcondor-stable-rhel6.repo
+```
+If you wish to use a [pool password for authentication](http://research.cs.wisc.edu/htcondor/manual/latest/3_6Security.html#SECTION00463400000000000000) you will need to create one first: ```condor_store_cred -f <path_to_htcondor_module>/files/pool_password```.
+
+### Examples
+`hiera` config examples can be found in the examples folder. They describe a minimal example of
+ - settings shared across different node types: `htcondor_common.yaml`
+ - settings for managers (nodes that run collector & negotiator daemons): `htcondor_manager.yaml`
+ - settings for schedulers: `htcondor_scheduler.yaml`
+ - settings for worker nodes: `htcondor_common.yaml`
+The examples assume class management in hiere by adding  `hiera_include('classes')` to the `site.pp`.
+Real life examples can be found in https://github.com/uobdic/UKI-SOUTHGRID-BRIS-HEP.
+
+## Custom machine/job attributes
+Sometimes it is necessary to create custom attributes for condor. Machine attributes can be used
+in job requirements (e.g. `HasMatLab = True`) and job attributes for job reporting/monitoring (e.g. `HEPSPEC06 = 14.00`).
+To specify the attributes in hiera simply add
+```
+htcondor::custom_attributes:
+  - HasMatLab: True
+  ...
+```
+and for job attributes
+```
+htcondor::custom_job_attributes:
+  - HEPSPEC06: 14.00
+  - CPUScaling: 1.04
+  ...
+```
+Although the use is identical, they are put into different places. `custom_attributes` end up added to the `STARTD_ATTRS`
+and `custom_job_attributes` are added to `STARTD_JOB_ATTRS`.
+
+##Limitations
+###General
+
+
+##Development
+
+###Contributing
+###Running tests
 Please run
 ```bundle exec rake validate && bundle exec rake lint && bundle exec rake spec SPEC_OPTS='--format documentation'```
 and make sure no errors are present when submitting code.
 
+### Generating changlelog
+```
+github_changelog_generator -u hep-puppet -p htcondor
+```

Rakefile

@@ -2,11 +2,12 @@ require 'rubygems'
 require 'puppetlabs_spec_helper/rake_tasks'
 require 'puppet-lint/tasks/puppet-lint'
 
-PuppetLint.configuration.fail_on_warnings
+PuppetLint.configuration.fail_on_warnings = true
 PuppetLint.configuration.send('relative')
 PuppetLint.configuration.send('disable_80chars')
 PuppetLint.configuration.send('disable_class_inherits_from_params_class')
 PuppetLint.configuration.send('disable_documentation')
 PuppetLint.configuration.send('disable_single_quote_string_with_variables')
 PuppetLint.configuration.send('disable_only_variable_string')
-PuppetLint.configuration.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp"]
+PuppetLint.configuration.ignore_paths = ["spec/**/*", "pkg/**/*", 'vendor/**/*']
+PuppetLint.configuration.exclude_paths = ["spec/**/*", "pkg/**/*", 'vendor/**/*']

examples/htcondor_common.yaml

@@ -0,0 +1,36 @@
+---
+message: "This is condor!"
+# common htcondor bits
+# all install condor
+classes:
+  - htcondor
+# common parameters
+htcondor::managers:
+  - collector1.example.org
+  - collector2.example.org
+
+htcondor::schedulers:
+  - sched1.example.org
+  - sched2.example.org
+
+htcondor::workers:
+  - '*.example.org'
+
+htcondor::uid_domain: 'example.org'
+htcondor::default_domain_name: 'example.org'
+htcondor::cluster_has_multiple_domains: true
+htcondor::collector_name: 'Example-HTCondor-Cluster'
+htcondor::cluster_has_multiple_domains: true
+htcondor::admin_email: localhost
+htcondor::condor_user: condor
+htcondor::condor_group: condor
+htcondor::machine_owner: condor
+htcondor::include_username_in_accounting: true
+
+htcondor::pool_home: '/condor'
+htcondor::use_pkg_condor_config: true
+# this refers to a custom puppet mount called 'secrets'
+htcondor::pool_password: 'puppet:///secrets/pool_password'
+
+# should be off for a production system (only true for fresh install or update is needed)
+htcondor::install_repositories: false