Skip to content

Commit 2a5c81f

Browse files
jmthomasjmthomas
authored
Merge pull request #678 from OpenC3/browse_volumes
Browse volumes
2 parents 111e58d + 71a6e01 commit 2a5c81f

File tree

5 files changed

+244
-62
lines changed

5 files changed

+244
-62
lines changed

‎.env

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,19 @@ OPENC3_NAMESPACE=openc3inc
1313
OPENC3_DEPENDENCY_REGISTRY=docker.io
1414
OPENC3_ENTERPRISE_REGISTRY=ghcr.io
1515
OPENC3_ENTERPRISE_NAMESPACE=openc3
16-
# Bucket configuration
16+
# Bucket & Volume configuration
1717
OPENC3_LOGS_BUCKET=logs
1818
OPENC3_TOOLS_BUCKET=tools
1919
OPENC3_CONFIG_BUCKET=config
20-
# Add additional buckets using the same convention:
20+
OPENC3_GEMS_VOLUME=/gems
21+
OPENC3_PLUGIN_DEFAULT_VOLUME=/plugins/DEFAULT
22+
# Add additional buckets and volumes using the same convention:
2123
# OPENC3_TEST_BUCKET=test_bucket
2224
# This would add a new bucket called "test". Note the middle section OPENC3_(TEST)_BUCKET
2325
# is the displayed name of the bucket and the value is the actual bucket name in MINIO, S3, GCP, etc.
26+
# OPENC3_TEST_VOLUME=/path/to/somewhere
27+
# This would add a new volume called "test". Note the middle section OPENC3_(TEST)_VOLUME
28+
# is the displayed name of the volume and the value is the actual root path of the volume.
2429

2530
# Redis configuration
2631
OPENC3_REDIS_HOSTNAME=openc3-redis

‎openc3-cosmos-cmd-tlm-api/app/controllers/storage_controller.rb

Lines changed: 120 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# encoding: ascii-8bit
1+
# encoding: utf-8
22

33
# Copyright 2022 Ball Aerospace & Technologies Corp.
44
# All Rights Reserved.
@@ -34,67 +34,169 @@ def buckets
3434
render :json => buckets, :status => 200
3535
end
3636

37+
def volumes
38+
# ENV.map returns a big array of mostly nils which is why we compact
39+
# The non-nil are MatchData objects due to the regex match
40+
matches = ENV.map { |key, value| key.match(/^OPENC3_(.+)_VOLUME$/) }.compact
41+
# MatchData [0] is the full text, [1] is the captured group
42+
# downcase to make it look nicer, BucketExplorer.vue calls toUpperCase on the API requests
43+
volumes = matches.map { |match| match[1].downcase }.sort
44+
# Add a slash prefix to identify volumes separately from buckets
45+
volumes.map! {|volume| "/#{volume}" }
46+
render :json => volumes, :status => 200
47+
end
48+
3749
def files
3850
return unless authorization('system')
39-
bucket = OpenC3::Bucket.getClient()
40-
bucket_name = ENV[params[:bucket]] # Get the actual bucket name
41-
path = params[:path]
42-
path = '/' if path.nil? || path.empty?
43-
# if user wants metadata returned
44-
metadata = params[:metadata].present? ? true : false
45-
results = bucket.list_files(bucket: bucket_name, path: path, metadata: metadata)
51+
root = ENV[params[:root]] # Get the actual bucket / volume name
52+
raise "Unknown bucket / volume #{params[:root]}" unless root
53+
results = []
54+
if params[:root].include?('_BUCKET')
55+
bucket = OpenC3::Bucket.getClient()
56+
path = sanitize_path(params[:path])
57+
path = '/' if path.empty?
58+
# if user wants metadata returned
59+
metadata = params[:metadata].present? ? true : false
60+
results = bucket.list_files(bucket: root, path: path, metadata: metadata)
61+
elsif params[:root].include?('_VOLUME')
62+
dirs = []
63+
files = []
64+
path = sanitize_path(params[:path])
65+
list = Dir["/#{root}/#{path}/*"] # Ok for path to be blank
66+
list.each do |file|
67+
if File.directory?(file)
68+
dirs << File.basename(file)
69+
else
70+
stat = File.stat(file)
71+
files << { name: File.basename(file), size: stat.size, modified: stat.mtime }
72+
end
73+
end
74+
results << dirs
75+
results << files
76+
else
77+
raise "Unknown root #{params[:root]}"
78+
end
4679
render :json => results, :status => 200
4780
rescue OpenC3::Bucket::NotFound => error
4881
render :json => { :status => 'error', :message => error.message }, :status => 404
82+
rescue Exception => e
83+
OpenC3::Logger.error("File listing failed: #{e.message}", user: user_info(request.headers['HTTP_AUTHORIZATION']))
84+
render :json => { status: 'error', message: e.message }, status: 500
85+
end
86+
87+
def download_file
88+
return unless authorization('system')
89+
volume = ENV[params[:volume]] # Get the actual volume name
90+
raise "Unknown volume #{params[:volume]}" unless volume
91+
filename = "/#{volume}/#{params[:object_id]}"
92+
filename = sanitize_path(filename)
93+
file = File.read(filename, mode: 'rb')
94+
render :json => { filename: params[:object_id], contents: Base64.encode64(file) }
95+
rescue Exception => e
96+
OpenC3::Logger.error("Download failed: #{e.message}", user: user_info(request.headers['HTTP_AUTHORIZATION']))
97+
render :json => { status: 'error', message: e.message }, status: 500
4998
end
5099

51100
def get_download_presigned_request
52101
return unless authorization('system')
53102
bucket = OpenC3::Bucket.getClient()
54103
bucket_name = ENV[params[:bucket]] # Get the actual bucket name
55-
bucket.check_object(bucket: bucket_name, key: params[:object_id])
104+
raise "Unknown bucket #{params[:bucket]}" unless bucket_name
105+
path = sanitize_path(params[:object_id])
106+
bucket.check_object(bucket: bucket_name, key: path)
56107
result = bucket.presigned_request(bucket: bucket_name,
57-
key: params[:object_id],
108+
key: path,
58109
method: :get_object,
59110
internal: params[:internal])
60111
render :json => result, :status => 201
112+
rescue Exception => e
113+
OpenC3::Logger.error("Download request failed: #{e.message}", user: user_info(request.headers['HTTP_AUTHORIZATION']))
114+
render :json => { status: 'error', message: e.message }, status: 500
61115
end
62116

63117
def get_upload_presigned_request
64118
return unless authorization('system_set')
65119
bucket_name = ENV[params[:bucket]] # Get the actual bucket name
66-
key_split = params[:object_id].to_s.split('/')
120+
raise "Unknown bucket #{params[:bucket]}" unless bucket_name
121+
path = sanitize_path(params[:object_id])
122+
key_split = path.split('/')
67123
# Anywhere other than config/SCOPE/targets_modified requires admin
68124
if !(params[:bucket] == 'OPENC3_CONFIG_BUCKET' && key_split[1] == 'targets_modified')
69125
return unless authorization('admin')
70126
end
71127

72128
bucket = OpenC3::Bucket.getClient()
73129
result = bucket.presigned_request(bucket: bucket_name,
74-
key: params[:object_id],
130+
key: path,
75131
method: :put_object,
76132
internal: params[:internal])
77-
OpenC3::Logger.info("S3 upload presigned request generated: #{bucket_name}/#{params[:object_id]}",
133+
OpenC3::Logger.info("S3 upload presigned request generated: #{bucket_name}/#{path}",
78134
scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
79135
render :json => result, :status => 201
136+
rescue Exception => e
137+
OpenC3::Logger.error("Upload request failed: #{e.message}", user: user_info(request.headers['HTTP_AUTHORIZATION']))
138+
render :json => { status: 'error', message: e.message }, status: 500
80139
end
81140

82141
def delete
83142
return unless authorization('system_set')
143+
if params[:bucket].presence
144+
deleteBucketItem(params)
145+
elsif params[:volume].presence
146+
deleteVolumeItem(params)
147+
else
148+
raise "Must pass bucket or volume parameter!"
149+
end
150+
head :ok
151+
rescue Exception => e
152+
OpenC3::Logger.error("Delete failed: #{e.message}", user: user_info(request.headers['HTTP_AUTHORIZATION']))
153+
render :json => { status: 'error', message: e.message }, status: 500
154+
end
155+
156+
private
157+
158+
def sanitize_path(path)
159+
return '' if path.nil?
160+
# path is passed as a parameter thus we have to sanitize it or the code scanner detects:
161+
# "Uncontrolled data used in path expression"
162+
# This method is taken directly from the Rails source:
163+
# https://api.rubyonrails.org/v5.2/classes/ActiveStorage/Filename.html#method-i-sanitized
164+
# NOTE: I removed the '/' character because we have to allow this in order to traverse the path
165+
sanitized = path.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�").strip.tr("\u{202E}%$|:;\t\r\n\\", "-").gsub('..', '-')
166+
if sanitized != path
167+
raise "Invalid path: #{path}"
168+
end
169+
sanitized
170+
end
171+
172+
def deleteBucketItem(params)
84173
bucket_name = ENV[params[:bucket]] # Get the actual bucket name
85-
key_split = params[:object_id].to_s.split('/')
174+
raise "Unknown bucket #{params[:bucket]}" unless bucket_name
175+
path = sanitize_path(params[:object_id])
176+
key_split = path.split('/')
86177
# Anywhere other than config/SCOPE/targets_modified requires admin
87178
if !(params[:bucket] == 'OPENC3_CONFIG_BUCKET' && key_split[1] == 'targets_modified')
88179
return unless authorization('admin')
89180
end
90181

91182
if ENV['OPENC3_LOCAL_MODE']
92-
OpenC3::LocalMode.delete_local(params[:object_id])
183+
OpenC3::LocalMode.delete_local(path)
93184
end
94185

95-
OpenC3::Bucket.getClient().delete_object(bucket: bucket_name, key: params[:object_id])
96-
OpenC3::Logger.info("Deleted: #{bucket_name}/#{params[:object_id]}",
186+
OpenC3::Bucket.getClient().delete_object(bucket: bucket_name, key: path)
187+
OpenC3::Logger.info("Deleted: #{bucket_name}/#{path}",
188+
scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
189+
end
190+
191+
def deleteVolumeItem(params)
192+
# Deleting requires admin
193+
return unless authorization('admin')
194+
volume = ENV[params[:volume]] # Get the actual volume name
195+
raise "Unknown volume #{params[:volume]}" unless volume
196+
filename = "/#{volume}/#{params[:object_id]}"
197+
filename = sanitize_path(filename)
198+
FileUtils.rm filename
199+
OpenC3::Logger.info("Deleted: #{filename}",
97200
scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
98-
head :ok
99201
end
100202
end

‎openc3-cosmos-cmd-tlm-api/config/routes.rb

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,9 @@
149149
get '/autocomplete/data/:type', to: 'script_autocomplete#get_ace_autocomplete_data', type: /[^\/]+/
150150

151151
get '/storage/buckets', to: 'storage#buckets'
152-
get '/storage/files/:bucket/(*path)', to: 'storage#files'
152+
get '/storage/volumes', to: 'storage#volumes'
153+
get '/storage/files/:root/(*path)', to: 'storage#files'
154+
get '/storage/download_file/:object_id', to: 'storage#download_file', object_id: /.*/
153155
get '/storage/download/:object_id', to: 'storage#get_download_presigned_request', object_id: /.*/
154156
get '/storage/upload/:object_id', to: 'storage#get_upload_presigned_request', object_id: /.*/
155157
delete '/storage/delete/:object_id', to: 'storage#delete', object_id: /.*/

‎openc3-cosmos-init/plugins/packages/openc3-cosmos-tool-bucketexplorer/src/router.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export default new Router({
2626
base: process.env.BASE_URL,
2727
routes: [
2828
{
29-
path: '/:path?',
29+
path: '/:path*',
3030
name: 'Bucket Explorer',
3131
component: () => import('./tools/BucketExplorer/BucketExplorer.vue'),
3232
},

0 commit comments

Comments
 (0)