Commit d0eaed5f authored by Jay Mundrawala's avatar Jay Mundrawala

Merge pull request #533 from chef/jdm/del-uber-s3-cache

Remove uber-s3 dependency for s3_cache
parents 64c47701 30dac29d
......@@ -51,6 +51,10 @@ module Omnibus
desc: 'The accessibility of the uploaded packages',
enum: %w(public private),
default: 'private'
method_option :region,
type: :string,
desc: 'The region in which the bucket is located',
default: 'us-east-1'
desc 's3 BUCKET PATTERN', 'Publish to an S3 bucket'
def s3(bucket, pattern)
options[:bucket] = bucket
......
......@@ -289,6 +289,14 @@ module Omnibus
raise MissingRequiredAttribute.new(self, :s3_secret_key, "'EFGH5678'")
end
# The region of the S3 bucket you want to cache software artifacts in.
# Defaults to 'us-east-1'
#
# @return [String]
default(:s3_region) do
'us-east-1'
end
# --------------------------------------------------
# @!endgroup
#
......
......@@ -14,11 +14,14 @@
# limitations under the License.
#
require 'omnibus/s3_helpers'
module Omnibus
class S3Publisher < Publisher
include S3Helpers
def publish(&block)
log.info(log_key) { 'Starting S3 publisher' }
safe_require('uber-s3')
packages.each do |package|
# Make sure the package is good to go!
......@@ -27,16 +30,13 @@ module Omnibus
# Upload the metadata first
log.debug(log_key) { "Uploading '#{package.metadata.name}'" }
client.store(key_for(package, package.metadata.name), package.metadata.to_json,
access: access_policy,
)
store_object(key_for(package, package.metadata.name), package.metadata.to_json,
nil, access_policy)
# Upload the actual package
log.info(log_key) { "Uploading '#{package.name}'" }
client.store(key_for(package, package.name), package.content,
access: access_policy,
content_md5: package.metadata[:md5],
)
store_object(key_for(package, package.name), package.content,
package.metadata[:md5], access_policy)
# If a block was given, "yield" the package to the caller
block.call(package) if block
......@@ -45,18 +45,13 @@ module Omnibus
private
#
# The actual S3 client object to communicate with the S3 API.
#
# @return [UberS3]
#
def client
@client ||= UberS3.new(
access_key: Config.publish_s3_access_key,
def s3_configuration
{
region: @options[:region],
access_key_id: Config.publish_s3_access_key,
secret_access_key: Config.publish_s3_secret_key,
bucket: @options[:bucket],
adaper: :net_http,
)
bucket_name: @options[:bucket],
}
end
#
......@@ -85,14 +80,14 @@ module Omnibus
# initializer option. Any access control that is not the strict string
# +"public"+ is assumed to be private.
#
# @return [Symbol]
# the UberS3-ready access policy
# @return [String]
# the access policy
#
def access_policy
if @options[:acl].to_s == 'public'
:public_read
'public-read'
else
:private
'private'
end
end
end
......
......@@ -15,13 +15,14 @@
#
require 'fileutils'
require 'uber-s3'
require 'omnibus/s3_helpers'
module Omnibus
class S3Cache
include Logging
class << self
include S3Helpers
#
# List all software in the cache.
#
......@@ -41,7 +42,7 @@ module Omnibus
# @return [Array<String>]
#
def keys
bucket.objects('/').map(&:key)
bucket.objects.map(&:key)
end
#
......@@ -70,16 +71,14 @@ module Omnibus
key = key_for(software)
fetcher = software.fetcher
content = IO.read(fetcher.downloaded_file)
log.info(log_key) do
"Caching '#{fetcher.downloaded_file}' to '#{Config.s3_bucket}/#{key}'"
end
client.store(key, content,
access: :public_read,
content_md5: software.fetcher.checksum
)
File.open(fetcher.downloaded_file, 'rb') do |file|
store_object(key, file, software.fetcher.checksum, 'public-read')
end
end
true
......@@ -129,33 +128,13 @@ module Omnibus
private
#
# The client to connect to S3 with.
#
# @return [UberS3::Client]
#
def client
@client ||= UberS3.new(
access_key: Config.s3_access_key,
secret_access_key: Config.s3_secret_key,
bucket: Config.s3_bucket,
adapter: :net_http,
)
end
#
# The bucket where the objects live.
#
# @return [UberS3::Bucket]
#
def bucket
@bucket ||= begin
if client.exists?('/')
client.bucket
else
client.connection.put('/')
end
end
def s3_configuration
{
region: Config.s3_region,
access_key_id: Config.s3_access_key,
secret_access_key: Config.s3_secret_key,
bucket_name: Config.s3_bucket
}
end
#
......
#
# Copyright 2015 Chef Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
require 'aws-sdk'
require 'base64'
module Omnibus
module S3Helpers
def self.included(base)
base.send(:include, InstanceMethods)
end
module InstanceMethods
private
#
# Returns the configuration for S3. You must provide keys
# :region,
#
# @example
# {
# region: 'us-east-1',
# access_key_id: Config.s3_access_key,
# secret_access_key: Config.s3_secret_key,
# bucket_name: Config.s3_bucket
# }
#
# @return [Hash<String, String>]
#
def s3_configuration
raise "You must override s3_configuration"
end
#
# The client to connect to S3 with.
#
# @return [Aws::S3::Resource]
#
def client
@s3_client ||= Aws::S3::Resource.new(
region: s3_configuration[:region],
access_key_id: s3_configuration[:access_key_id],
secret_access_key: s3_configuration[:secret_access_key]
)
end
#
# The bucket where the objects live.
#
# @return [Aws::S3::Bucket]
#
def bucket
@s3_bucket ||= begin
bucket = client.bucket(s3_configuration[:bucket_name])
unless bucket.exists?
bucket_config = if s3_configuration[:region] == 'us-east-1'
nil
else
{
location_constraint: s3_configuration[:region]
}
end
bucket.create(create_bucket_configuration: bucket_config)
end
bucket
end
end
#
# Store an object at the specified key
#
# @param [String] key
# @param [File, String] content
# @param [String] content_md5
# @param [String] acl
#
# @return [true]
#
def store_object(key, content, content_md5, acl)
bucket.put_object({
key: key,
body: content,
content_md5: to_base64_digest(content_md5),
acl: acl
})
true
end
#
# Convert a hex digest into a base64 hex digest
#
# For example:
# to_base64_digest('c3b5247592ce694f7097873aa07d66fe') => 'w7UkdZLOaU9wl4c6oH1m/g=='
#
# @param [String] content_md5
#
# @return [String]
#
def to_base64_digest(content_md5)
if content_md5
md5_digest = content_md5.unpack('a2'*16).collect {|i| i.hex.chr }.join
Base64.encode64(md5_digest).strip
end
end
end
end
end
......@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
gem.add_dependency 'mixlib-versioning'
gem.add_dependency 'ohai', '~> 8.0'
gem.add_dependency 'ruby-progressbar', '~> 1.7'
gem.add_dependency 'uber-s3', '~> 0.2.4'
gem.add_dependency 'aws-sdk', '~> 2'
gem.add_dependency 'thor', '~> 0.18'
gem.add_development_dependency 'bundler'
......
......@@ -30,11 +30,12 @@ module Omnibus
let(:packages) { [package] }
let(:client) { double('UberS3', store: nil) }
let(:client) { double('Aws::S3::Resource') }
before do
allow(package).to receive(:metadata).and_return(metadata)
allow(subject).to receive(:client).and_return(client)
allow(subject).to receive(:store_object)
end
subject { described_class.new(path) }
......@@ -48,21 +49,22 @@ module Omnibus
end
it 'uploads the metadata' do
expect(client).to receive(:store).with(
expect(subject).to receive(:store_object).with(
'ubuntu/14.04/x86_64/chef.deb/chef.deb.metadata.json',
package.metadata.to_json,
access: :private,
nil,
'private',
).once
subject.publish
end
it 'uploads the package' do
expect(client).to receive(:store).with(
expect(subject).to receive(:store_object).with(
'ubuntu/14.04/x86_64/chef.deb/chef.deb',
package.content,
access: :private,
content_md5: package.metadata[:md5],
package.metadata[:md5],
'private'
).once
subject.publish
......@@ -72,10 +74,11 @@ module Omnibus
subject { described_class.new(path, acl: 'public') }
it 'sets the access control to public_read' do
expect(client).to receive(:store).with(
expect(subject).to receive(:store_object).with(
'ubuntu/14.04/x86_64/chef.deb/chef.deb.metadata.json',
package.metadata.to_json,
access: :public_read,
nil,
'public-read',
).once
subject.publish
......@@ -86,10 +89,11 @@ module Omnibus
subject { described_class.new(path, acl: 'baconbits') }
it 'sets the access control to private' do
expect(client).to receive(:store).with(
expect(subject).to receive(:store_object).with(
'ubuntu/14.04/x86_64/chef.deb/chef.deb.metadata.json',
package.metadata.to_json,
access: :private,
nil,
'private',
).once
subject.publish
......
......@@ -43,7 +43,7 @@ module Omnibus
before { allow(S3Cache).to receive(:bucket).and_return(bucket) }
it 'lists the keys on the S3 bucket' do
expect(bucket).to receive(:objects).with('/').once
expect(bucket).to receive(:objects).once
S3Cache.keys
end
end
......
require 'spec_helper'
require 'omnibus/s3_helpers'
module Omnibus
describe S3Helpers do
include Omnibus::S3Helpers
context 'when #s3_configuration is not defined' do
describe '#client' do
it 'raises an error if it is not overridden' do
expect { s3_configuration }.to raise_error(RuntimeError,
"You must override s3_configuration")
end
it 'raises an error stating that s3_configuration must be overriden' do
expect { client }.to raise_error(RuntimeError,
"You must override s3_configuration")
end
end
end
describe '#to_base64_digest' do
it 'turns "c3b5247592ce694f7097873aa07d66fe" into "w7UkdZLOaU9wl4c6oH1m/g=="' do
expect(to_base64_digest("c3b5247592ce694f7097873aa07d66fe")).to eql('w7UkdZLOaU9wl4c6oH1m/g==')
end
it 'allows a nil input without error' do
expect(to_base64_digest(nil)).to be_nil
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment